From ca5194625abdb2acb90ac3e975b5c57b6a6a96a3 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Fri, 9 May 2014 11:24:14 -0700 Subject: [PATCH 001/113] Add packaging skeleton for release. --- .gitignore | 44 +++- CHANGELOG.md | 5 + LICENSE | 17 ++ MANIFEST.in | 9 + Makefile | 57 +++++ README.md | 12 ++ docs/Makefile | 177 +++++++++++++++ docs/conf.py | 275 ++++++++++++++++++++++++ docs/index.rst | 23 ++ docs/installation.rst | 12 ++ docs/make.bat | 242 +++++++++++++++++++++ docs/usage.rst | 7 + __init__.py => gnsq/__init__.py | 0 backofftimer.py => gnsq/backofftimer.py | 0 errors.py => gnsq/errors.py | 0 httpclient.py => gnsq/httpclient.py | 0 lookupd.py => gnsq/lookupd.py | 0 message.py => gnsq/message.py | 0 nsqd.py => gnsq/nsqd.py | 0 protocal.py => gnsq/protocal.py | 0 reader.py => gnsq/reader.py | 0 util.py => gnsq/util.py | 0 requirements.txt | 6 + setup.cfg | 2 + setup.py | 39 ++++ tests/__init__.py | 2 + tests/test_gnsq.py | 28 +++ tox.ini | 9 + 28 files changed, 965 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 README.md create mode 100644 docs/Makefile create mode 100755 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/installation.rst create mode 100644 docs/make.bat create mode 100644 docs/usage.rst rename __init__.py => gnsq/__init__.py (100%) rename backofftimer.py => gnsq/backofftimer.py (100%) rename errors.py => gnsq/errors.py (100%) rename httpclient.py => gnsq/httpclient.py (100%) rename lookupd.py => gnsq/lookupd.py (100%) rename message.py => gnsq/message.py (100%) rename nsqd.py => gnsq/nsqd.py (100%) rename protocal.py => gnsq/protocal.py (100%) rename reader.py => gnsq/reader.py (100%) rename util.py => gnsq/util.py (100%) create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100755 setup.py create mode 100755 tests/__init__.py create mode 100755 tests/test_gnsq.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 0d20b64..74ffd4e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,43 @@ -*.pyc +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +htmlcov + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +docs/_build diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5d7dea7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## 0.0.1 - + +Initial release. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..89de354 --- /dev/null +++ b/LICENSE @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..edbfa1e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +include CHANGELOG.md +include LICENSE +include README.md + +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.rst conf.py Makefile make.bat diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8ef6348 --- /dev/null +++ b/Makefile @@ -0,0 +1,57 @@ +.PHONY: clean-pyc clean-build docs clean + +help: + @echo "clean-build - remove build artifacts" + @echo "clean-pyc - remove Python file artifacts" + @echo "lint - check style with flake8" + @echo "test - run tests quickly with the default Python" + @echo "test-all - run tests on every Python version with tox" + @echo "coverage - check code coverage quickly with the default Python" + @echo "docs - generate Sphinx HTML documentation, including API docs" + @echo "release - package and upload a release" + @echo "dist - package" + +clean: clean-build clean-pyc + rm -fr htmlcov/ + +clean-build: + rm -fr build/ + rm -fr dist/ + rm -fr *.egg-info + +clean-pyc: + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + +lint: + flake8 gnsq tests + +test: + python setup.py test + +test-all: + tox + +coverage: + coverage run --source gnsq setup.py test + coverage report -m + coverage html + open htmlcov/index.html + +docs: + rm -f docs/gnsq.rst + rm -f docs/modules.rst + sphinx-apidoc -o docs/ gnsq + $(MAKE) -C docs clean + $(MAKE) -C docs html + open docs/_build/html/index.html + +release: clean + python setup.py sdist upload + python setup.py bdist_wheel upload + +dist: clean + python setup.py sdist + python setup.py bdist_wheel + ls -l dist diff --git a/README.md b/README.md new file mode 100644 index 0000000..80aa98f --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +gnsq +==== + +A gevent based NSQ driver for Python. + +* Free software: MIT license +* Documentation: http://gnsq.rtfd.org. + +Features +-------- + +* TODO diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..0e35bee --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,177 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/complexity.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/complexity.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/complexity" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/complexity" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/conf.py b/docs/conf.py new file mode 100755 index 0000000..ef0540f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# complexity documentation build configuration file, created by +# sphinx-quickstart on Tue Jul 9 22:26:36 2013. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another +# directory, add these directories to sys.path here. If the directory is +# relative to the documentation root, use os.path.abspath to make it +# absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# Get the project root dir, which is the parent dir of this +cwd = os.getcwd() +project_root = os.path.dirname(cwd) + +# Insert the project root dir as the first element in the PYTHONPATH. +# This lets us ensure that the source package is imported, and that its +# version is used. +sys.path.insert(0, project_root) + +import gnsq + +# -- General configuration --------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'gnsq' +copyright = u'2014, William Trevor Olson' + +# The version info for the project you're documenting, acts as replacement +# for |version| and |release|, also used in various other places throughout +# the built documents. +# +# The short X.Y version. +version = gnsq.__version__ +# The full version, including alpha/beta/rc tags. +release = gnsq.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to +# some non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built +# documents. +#keep_warnings = False + + +# -- Options for HTML output ------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a +# theme further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as +# html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the +# top of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon +# of the docs. This file should be a Windows icon file (.ico) being +# 16x16 or 32x32 pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) +# here, relative to this directory. They are copied after the builtin +# static files, so a file named "default.css" will overwrite the builtin +# "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names +# to template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. +# Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. +# Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages +# will contain a tag referring to it. The value of this option +# must be the base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'gnsqdoc' + + +# -- Options for LaTeX output ------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + #'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', 'gnsq.tex', + u'gnsq Documentation', + u'William Trevor Olson', 'manual'), +] + +# The name of an image file (relative to this directory) to place at +# the top of the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings +# are parts, not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output ------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'gnsq', + u'gnsq Documentation', + [u'William Trevor Olson'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ---------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'gnsq', + u'gnsq Documentation', + u'William Trevor Olson', + 'gnsq', + 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..2711ce0 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,23 @@ +.. complexity documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to gnsq's documentation! +====================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + installation + usage + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..90f1c48 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,12 @@ +============ +Installation +============ + +At the command line:: + + $ easy_install gnsq + +Or, if you have virtualenvwrapper installed:: + + $ mkvirtualenv gnsq + $ pip install gnsq diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..fec43bb --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,242 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\complexity.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\complexity.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 0000000..e524b38 --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,7 @@ +======== +Usage +======== + +To use gnsq in a project:: + + import gnsq diff --git a/__init__.py b/gnsq/__init__.py similarity index 100% rename from __init__.py rename to gnsq/__init__.py diff --git a/backofftimer.py b/gnsq/backofftimer.py similarity index 100% rename from backofftimer.py rename to gnsq/backofftimer.py diff --git a/errors.py b/gnsq/errors.py similarity index 100% rename from errors.py rename to gnsq/errors.py diff --git a/httpclient.py b/gnsq/httpclient.py similarity index 100% rename from httpclient.py rename to gnsq/httpclient.py diff --git a/lookupd.py b/gnsq/lookupd.py similarity index 100% rename from lookupd.py rename to gnsq/lookupd.py diff --git a/message.py b/gnsq/message.py similarity index 100% rename from message.py rename to gnsq/message.py diff --git a/nsqd.py b/gnsq/nsqd.py similarity index 100% rename from nsqd.py rename to gnsq/nsqd.py diff --git a/protocal.py b/gnsq/protocal.py similarity index 100% rename from protocal.py rename to gnsq/protocal.py diff --git a/reader.py b/gnsq/reader.py similarity index 100% rename from reader.py rename to gnsq/reader.py diff --git a/util.py b/gnsq/util.py similarity index 100% rename from util.py rename to gnsq/util.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..95994dc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +blinker==1.3 +gevent==1.0.1 +gnsq==0.0.1 +greenlet==0.4.2 +requests==2.2.1 +wheel==0.22.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5e40900 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[wheel] +universal = 1 diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..ceb03f9 --- /dev/null +++ b/setup.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys + + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +if sys.argv[-1] == 'publish': + os.system('python setup.py sdist upload') + sys.exit() + + +setup( + name='gnsq', + version='0.0.1', + description='A gevent based NSQ driver for Python.', + long_description=open('README.md').read(), + author='William Trevor Olson', + author_email='trevor@heytrevor.com', + url='https://github.com/wtolson/gnsq', + packages=['gnsq'], + include_package_data=True, + install_requires=[ + 'gevent', + 'blinker', + 'requests', + ], + license="MIT", + zip_safe=False, + classifiers=[ + 'License :: OSI Approved :: MIT License', + ], + test_suite='tests', +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100755 index 0000000..faa18be --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- diff --git a/tests/test_gnsq.py b/tests/test_gnsq.py new file mode 100755 index 0000000..9599d9b --- /dev/null +++ b/tests/test_gnsq.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +test_gnsq +---------------------------------- + +Tests for `gnsq` module. +""" + +import unittest + +from gnsq import nsq + + +class TestGnsq(unittest.TestCase): + + def setUp(self): + pass + + def test_something(self): + pass + + def tearDown(self): + pass + +if __name__ == '__main__': + unittest.main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..ec49931 --- /dev/null +++ b/tox.ini @@ -0,0 +1,9 @@ +[tox] +envlist = py27, py34 + +[testenv] +setenv = + PYTHONPATH = {toxinidir}:{toxinidir}/gnsq +commands = python setup.py test +deps = + -r{toxinidir}/requirements.txt From 0215fe975750a6046d5e2d7c231f60d382507dd1 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 11 May 2014 11:55:29 -0700 Subject: [PATCH 002/113] Yay pep8. --- gnsq/__init__.py | 16 ++++++++++++---- gnsq/backofftimer.py | 3 ++- gnsq/errors.py | 27 +++++++++++++++++++++++---- gnsq/httpclient.py | 1 + gnsq/lookupd.py | 1 + gnsq/nsqd.py | 16 ++++++++-------- gnsq/protocal.py | 19 +++++++++++++++++++ gnsq/reader.py | 6 +++--- 8 files changed, 69 insertions(+), 20 deletions(-) diff --git a/gnsq/__init__.py b/gnsq/__init__.py index b3073e8..f558a84 100644 --- a/gnsq/__init__.py +++ b/gnsq/__init__.py @@ -1,5 +1,13 @@ -from .reader import Reader -from .nsqd import Nsqd -from .lookupd import Lookupd -from .message import Message +from .reader import Reader +from .nsqd import Nsqd +from .lookupd import Lookupd +from .message import Message from .backofftimer import BackoffTimer + +__all__ = [ + 'Reader', + 'Nsqd', + 'Lookupd', + 'Message', + 'BackoffTimer', +] diff --git a/gnsq/backofftimer.py b/gnsq/backofftimer.py index 1f3911c..bee4740 100644 --- a/gnsq/backofftimer.py +++ b/gnsq/backofftimer.py @@ -1,5 +1,6 @@ from random import randint + class BackoffTimer(object): def __init__(self, ratio=1, max_interval=None, min_interval=None): self.c = 0 @@ -13,7 +14,7 @@ def reset(self): return self def success(self): - self.c = max(self.c-1, 0) + self.c = max(self.c - 1, 0) return self def failure(self): diff --git a/gnsq/errors.py b/gnsq/errors.py index 2424882..b61e4c2 100644 --- a/gnsq/errors.py +++ b/gnsq/errors.py @@ -1,64 +1,82 @@ import socket + class NSQException(Exception): pass + class NSQRequeueMessage(NSQException): pass + class NSQNoConnections(NSQException): pass + class NSQSocketError(socket.error, NSQException): pass + class NSQFrameError(NSQException): pass + class NSQErrorCode(NSQException): - pass + fatal = True + class NSQInvalid(NSQErrorCode): """E_INVALID""" pass + class NSQBadBody(NSQErrorCode): """E_BAD_BODY""" pass + class NSQBadTopic(NSQErrorCode): """E_BAD_TOPIC""" pass + class NSQBadChannel(NSQErrorCode): """E_BAD_CHANNEL""" pass + class NSQBadMessage(NSQErrorCode): """E_BAD_MESSAGE""" pass + class NSQPutFailed(NSQErrorCode): """E_PUT_FAILED""" pass + class NSQPubFailed(NSQErrorCode): """E_PUB_FAILED""" + class NSQMPubFailed(NSQErrorCode): """E_MPUB_FAILED""" + class NSQFinishFailed(NSQErrorCode): """E_FIN_FAILED""" - pass + fatal = False + class NSQRequeueFailed(NSQErrorCode): """E_REQ_FAILED""" - pass + fatal = False + class NSQTouchFailed(NSQErrorCode): """E_TOUCH_FAILED""" - pass + fatal = False + ERROR_CODES = { 'E_INVALID': NSQInvalid, @@ -76,5 +94,6 @@ class NSQTouchFailed(NSQErrorCode): 'E_TOUCH_FAILED': NSQTouchFailed } + def make_error(error_code): return ERROR_CODES.get(error_code, NSQErrorCode)(error_code) diff --git a/gnsq/httpclient.py b/gnsq/httpclient.py index 83a4d0b..7b72e9b 100644 --- a/gnsq/httpclient.py +++ b/gnsq/httpclient.py @@ -1,6 +1,7 @@ import requests from . import errors + class HTTPClient(object): base_url = None _session = None diff --git a/gnsq/lookupd.py b/gnsq/lookupd.py index 25e38ab..02a9dda 100644 --- a/gnsq/lookupd.py +++ b/gnsq/lookupd.py @@ -1,6 +1,7 @@ from .httpclient import HTTPClient from . import protocal as nsq + class Lookupd(HTTPClient): def __init__(self, address='http://localhost:4161/'): if not address.endswith('/'): diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index e64da16..9176802 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -1,7 +1,6 @@ -import re import blinker - import gevent + from gevent import socket from gevent.queue import Queue from gevent.event import AsyncResult @@ -16,6 +15,7 @@ HOSTNAME = socket.gethostname() SHORTNAME = HOSTNAME.split('.')[0] + class Nsqd(HTTPClient): def __init__(self, address = '127.0.0.1', @@ -51,7 +51,7 @@ def is_connected(self): def connect(self): if self.is_connected: - raise NSQException('already connected') + raise errors.NSQException('already connected') self.reset() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -358,11 +358,11 @@ def __hash__(self): return hash((self.address, self.tcp_port)) def __eq__(self, other): - return ( - isinstance(other, Nsqd) and - self.address == other.address and - self.tcp_port == other.tcp_port - ) + return all([ + isinstance(other, Nsqd), + self.address == other.address, + self.tcp_port == other.tcp_port, + ]) def __cmp__(self, other): return hash(self) - hash(other) diff --git a/gnsq/protocal.py b/gnsq/protocal.py index a3c1ceb..9f62c2b 100644 --- a/gnsq/protocal.py +++ b/gnsq/protocal.py @@ -33,22 +33,27 @@ TOPIC_NAME_RE = re.compile(r'^[\.a-zA-Z0-9_-]+$') CHANNEL_NAME_RE = re.compile(r'^[\.a-zA-Z0-9_-]+(#ephemeral)?$') + def valid_topic_name(topic): if not 0 < len(topic) < 33: return False return bool(TOPIC_NAME_RE.match(topic)) + def valid_channel_name(channel): if not 0 < len(channel) < 33: return False return bool(CHANNEL_NAME_RE.match(channel)) + def assert_valid_topic_name(topic): assert valid_topic_name(topic) + def assert_valid_channel_name(channel): assert valid_channel_name(channel) + # # Responses # @@ -56,9 +61,11 @@ def unpack_size(data): assert len(data) == 4 return struct.unpack('>l', data)[0] + def unpack_response(data): return unpack_size(data[:4]), data[4:] + def unpack_message(data): timestamp = struct.unpack('>q', data[:8])[0] attempts = struct.unpack('>h', data[8:10])[0] @@ -66,6 +73,7 @@ def unpack_message(data): body = data[26:] return timestamp, attempts, message_id, body + # # Commands # @@ -74,43 +82,54 @@ def _packbody(body): return '' return struct.pack('>l', len(body)) + body + def _command(cmd, body, *params): return ''.join((' '.join((cmd,) + params), NEWLINE, _packbody(body))) + def identify(data): return _command('IDENTIFY', json.dumps(data)) + def subscribe(topic_name, channel_name): assert_valid_topic_name(topic_name) assert_valid_channel_name(channel_name) return _command('SUB', None, topic_name, channel_name) + def publish(topic_name, data): assert_valid_topic_name(topic_name) return _command('PUB', data, topic_name) + def multipublish(topic_name, messages): assert_valid_topic_name(topic_name) data = ''.join(_packbody(m) for m in messages) return _command('MPUB', data, topic_name) + def ready(count): assert isinstance(count, int), "ready count must be an integer" assert count >= 0, "ready count cannot be negative" return _command('RDY', None, str(count)) + def finish(message_id): return _command('FIN', None, message_id) + def requeue(message_id, timeout=0): assert isinstance(timeout, int), "requeue timeout must be an integer" return _command('REQ', None, message_id, str(timeout)) + def touch(message_id): return _command('TOUCH', None, message_id) + def close(): return _command('CLS', None) + def nop(): return _command('NOP', None) diff --git a/gnsq/reader.py b/gnsq/reader.py index 8112387..7810af8 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -4,8 +4,8 @@ import blinker from .lookupd import Lookupd -from .nsqd import Nsqd -from .util import assert_list +from .nsqd import Nsqd +from .util import assert_list from .errors import ( NSQException, @@ -124,7 +124,7 @@ def get_stats(self, conn): return None def smallest_depth(self): - if len(conn) == 0: + if len(self.conns) == 0: return None stats = self.stats From a76aaef79e8250e0517216f00094ad2505dad00f Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 11 May 2014 15:54:28 -0700 Subject: [PATCH 003/113] Consistent spacing. Use exceptions instead of asserts. --- gnsq/errors.py | 8 ++++++++ gnsq/message.py | 20 +++++++++++++------- gnsq/nsqd.py | 31 ++++++++++++++++++------------- gnsq/protocal.py | 42 ++++++++++++++++++++++++++++-------------- gnsq/reader.py | 46 ++++++++++++++++++++++------------------------ gnsq/util.py | 6 +++++- 6 files changed, 94 insertions(+), 59 deletions(-) diff --git a/gnsq/errors.py b/gnsq/errors.py index b61e4c2..b9293d3 100644 --- a/gnsq/errors.py +++ b/gnsq/errors.py @@ -5,6 +5,14 @@ class NSQException(Exception): pass +class NSQInvalidTopic(NSQException): + pass + + +class NSQInvalidChannel(NSQException): + pass + + class NSQRequeueMessage(NSQException): pass diff --git a/gnsq/message.py b/gnsq/message.py index 2ed7f24..e95a1c1 100644 --- a/gnsq/message.py +++ b/gnsq/message.py @@ -1,25 +1,31 @@ +from .errors import NSQException + + class Message(object): def __init__(self, conn, timestamp, attempts, id, body): self._has_responded = False - self.conn = conn + self.conn = conn self.timestamp = timestamp - self.attempts = attempts - self.id = id - self.body = body + self.attempts = attempts + self.id = id + self.body = body def has_responded(self): return self._has_responded def finish(self): - assert not self._has_responded + if self._has_responded: + raise NSQException('already responded') self._has_responded = True self.conn.finish(self.id) def requeue(self, time_ms=0): - assert not self._has_responded + if self._has_responded: + raise NSQException('already responded') self._has_responded = True self.conn.requeue(self.id, time_ms) def touch(self): - assert not self._has_responded + if self._has_responded: + raise NSQException('already responded') self.conn.touch(self.id) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 9176802..2d24b2d 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -23,24 +23,33 @@ def __init__(self, http_port = 4151, timeout = 60.0 ): + if not isinstance(address, (str, unicode)): + raise errors.NSQException('address must be a string') + + if not isinstance(tcp_port, int): + raise errors.NSQException('tcp_port must be a int') + + if not isinstance(http_port, int): + raise errors.NSQException('http_port must be a int') + self.address = address self.tcp_port = tcp_port self.http_port = http_port self.timeout = timeout self.on_response = blinker.Signal() - self.on_error = blinker.Signal() - self.on_message = blinker.Signal() - self.on_finish = blinker.Signal() - self.on_requeue = blinker.Signal() + self.on_error = blinker.Signal() + self.on_message = blinker.Signal() + self.on_finish = blinker.Signal() + self.on_requeue = blinker.Signal() self._send_worker = None - self._send_queue = Queue() + self._send_queue = Queue() self._frame_handlers = { nsq.FRAME_TYPE_RESPONSE: self.handle_response, - nsq.FRAME_TYPE_ERROR: self.handle_error, - nsq.FRAME_TYPE_MESSAGE: self.handle_message + nsq.FRAME_TYPE_ERROR: self.handle_error, + nsq.FRAME_TYPE_MESSAGE: self.handle_message } self.reset() @@ -355,14 +364,10 @@ def __str__(self): return self.address + ':' + str(self.tcp_port) def __hash__(self): - return hash((self.address, self.tcp_port)) + return hash(str(self)) def __eq__(self, other): - return all([ - isinstance(other, Nsqd), - self.address == other.address, - self.tcp_port == other.tcp_port, - ]) + return isinstance(other, Nsqd) and str(self) == str(other) def __cmp__(self, other): return hash(self) - hash(other) diff --git a/gnsq/protocal.py b/gnsq/protocal.py index 9f62c2b..466e3eb 100644 --- a/gnsq/protocal.py +++ b/gnsq/protocal.py @@ -1,6 +1,12 @@ import re import struct -import json + +try: + import simplejson as json +except ImportError: + import json # pyflakes.ignore + +from .errors import NSQException, NSQInvalidTopic, NSQInvalidChannel __all__ = [ 'MAGIC_V2', @@ -20,17 +26,17 @@ ] MAGIC_V2 = ' V2' -NEWLINE = '\n' +NEWLINE = '\n' FRAME_TYPE_RESPONSE = 0 -FRAME_TYPE_ERROR = 1 -FRAME_TYPE_MESSAGE = 2 +FRAME_TYPE_ERROR = 1 +FRAME_TYPE_MESSAGE = 2 # # Helpers # -TOPIC_NAME_RE = re.compile(r'^[\.a-zA-Z0-9_-]+$') +TOPIC_NAME_RE = re.compile(r'^[\.a-zA-Z0-9_-]+$') CHANNEL_NAME_RE = re.compile(r'^[\.a-zA-Z0-9_-]+(#ephemeral)?$') @@ -47,18 +53,21 @@ def valid_channel_name(channel): def assert_valid_topic_name(topic): - assert valid_topic_name(topic) + if valid_topic_name(topic): + return + raise NSQInvalidTopic() def assert_valid_channel_name(channel): - assert valid_channel_name(channel) + if valid_channel_name(channel): + return + raise NSQInvalidChannel() # # Responses # def unpack_size(data): - assert len(data) == 4 return struct.unpack('>l', data)[0] @@ -67,10 +76,10 @@ def unpack_response(data): def unpack_message(data): - timestamp = struct.unpack('>q', data[:8])[0] - attempts = struct.unpack('>h', data[8:10])[0] + timestamp = struct.unpack('>q', data[:8])[0] + attempts = struct.unpack('>h', data[8:10])[0] message_id = data[10:26] - body = data[26:] + body = data[26:] return timestamp, attempts, message_id, body @@ -109,8 +118,12 @@ def multipublish(topic_name, messages): def ready(count): - assert isinstance(count, int), "ready count must be an integer" - assert count >= 0, "ready count cannot be negative" + if not isinstance(count, int): + raise NSQException('ready count must be an integer') + + if count < 0: + raise NSQException('ready count cannot be negative') + return _command('RDY', None, str(count)) @@ -119,7 +132,8 @@ def finish(message_id): def requeue(message_id, timeout=0): - assert isinstance(timeout, int), "requeue timeout must be an integer" + if not isinstance(timeout, int): + raise NSQException('requeue timeout must be an integer') return _command('REQ', None, message_id, str(timeout)) diff --git a/gnsq/reader.py b/gnsq/reader.py index 7810af8..8f283a0 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -18,33 +18,35 @@ class Reader(object): def __init__(self, topic, channel, - nsqd_tcp_addresses = [], + nsqd_tcp_addresses = [], lookupd_http_addresses = [], - async = False, - max_tries = 5, - max_in_flight = 1, - lookupd_poll_interval = 120, - requeue_delay = 0 + async = False, + max_tries = 5, + max_in_flight = 1, + lookupd_poll_interval = 120, + requeue_delay = 0 ): - lookupd_http_addresses = assert_list(lookupd_http_addresses) - self.lookupds = [Lookupd(a) for a in lookupd_http_addresses] + lookupd_http_addresses = assert_list(lookupd_http_addresses) + self.lookupds = [Lookupd(a) for a in lookupd_http_addresses] self.nsqd_tcp_addresses = assert_list(nsqd_tcp_addresses) - assert self.nsqd_tcp_addresses or self.lookupds - self.topic = topic - self.channel = channel - self.async = async - self.max_tries = max_tries - self.max_in_flight = max_in_flight + if not self.nsqd_tcp_addresses and not self.lookupds: + raise NSQException('must specify at least on nsqd or lookupd') + + self.topic = topic + self.channel = channel + self.async = async + self.max_tries = max_tries + self.max_in_flight = max_in_flight self.lookupd_poll_interval = lookupd_poll_interval - self.requeue_delay = requeue_delay - self.logger = logging.getLogger(__name__) + self.requeue_delay = requeue_delay + self.logger = logging.getLogger(__name__) self.on_response = blinker.Signal() - self.on_error = blinker.Signal() - self.on_message = blinker.Signal() - self.on_finish = blinker.Signal() - self.on_requeue = blinker.Signal() + self.on_error = blinker.Signal() + self.on_message = blinker.Signal() + self.on_finish = blinker.Signal() + self.on_requeue = blinker.Signal() self.conns = set() self.stats = {} @@ -145,10 +147,6 @@ def publish(self, topic, message): conn.publish(topic, message) def connect_to_nsqd(self, address, tcp_port, http_port=None): - assert isinstance(address, (str, unicode)) - assert isinstance(tcp_port, int) - assert isinstance(http_port, int) or http_port is None - conn = Nsqd(address, tcp_port, http_port) if conn in self.conns: self.logger.debug('[%s] already connected' % conn) diff --git a/gnsq/util.py b/gnsq/util.py index d747044..b0a571c 100644 --- a/gnsq/util.py +++ b/gnsq/util.py @@ -1,7 +1,11 @@ +from .errors import NSQException + def assert_list(item): if isinstance(item, basestring): item = [item] - assert isinstance(item, (list, set, tuple)) + elif not isinstance(item, (list, set, tuple)): + raise NSQException('must be a list, set or tuple') + return item From acdab617f54950fef448acd1bd61679206986eb2 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 11 May 2014 16:03:09 -0700 Subject: [PATCH 004/113] More spacing fixes. --- gnsq/backofftimer.py | 2 +- gnsq/errors.py | 22 +++++++++++----------- gnsq/httpclient.py | 6 +++--- gnsq/nsqd.py | 42 +++++++++++++++++++++--------------------- gnsq/protocal.py | 1 + gnsq/reader.py | 6 +++--- 6 files changed, 40 insertions(+), 39 deletions(-) diff --git a/gnsq/backofftimer.py b/gnsq/backofftimer.py index bee4740..c12a53c 100644 --- a/gnsq/backofftimer.py +++ b/gnsq/backofftimer.py @@ -3,7 +3,7 @@ class BackoffTimer(object): def __init__(self, ratio=1, max_interval=None, min_interval=None): - self.c = 0 + self.c = 0 self.ratio = ratio self.max_interval = max_interval diff --git a/gnsq/errors.py b/gnsq/errors.py index b9293d3..d1442a4 100644 --- a/gnsq/errors.py +++ b/gnsq/errors.py @@ -87,19 +87,19 @@ class NSQTouchFailed(NSQErrorCode): ERROR_CODES = { - 'E_INVALID': NSQInvalid, - 'E_BAD_BODY': NSQBadBody, - 'E_BAD_TOPIC': NSQBadTopic, - 'E_BAD_CHANNEL': NSQBadChannel, - 'E_BAD_MESSAGE': NSQBadMessage, - 'E_PUT_FAILED': NSQPutFailed, - 'E_PUB_FAILED': NSQPubFailed, - 'E_MPUB_FAILED': NSQMPubFailed, + 'E_INVALID': NSQInvalid, + 'E_BAD_BODY': NSQBadBody, + 'E_BAD_TOPIC': NSQBadTopic, + 'E_BAD_CHANNEL': NSQBadChannel, + 'E_BAD_MESSAGE': NSQBadMessage, + 'E_PUT_FAILED': NSQPutFailed, + 'E_PUB_FAILED': NSQPubFailed, + 'E_MPUB_FAILED': NSQMPubFailed, 'E_FINISH_FAILED': NSQFinishFailed, - 'E_FIN_FAILED': NSQFinishFailed, + 'E_FIN_FAILED': NSQFinishFailed, 'E_REQUEUE_FAILED': NSQRequeueFailed, - 'E_REQ_FAILED': NSQRequeueFailed, - 'E_TOUCH_FAILED': NSQTouchFailed + 'E_REQ_FAILED': NSQRequeueFailed, + 'E_TOUCH_FAILED': NSQTouchFailed } diff --git a/gnsq/httpclient.py b/gnsq/httpclient.py index 7b72e9b..1e98bde 100644 --- a/gnsq/httpclient.py +++ b/gnsq/httpclient.py @@ -1,5 +1,5 @@ import requests -from . import errors +from .errors import NSQException class HTTPClient(object): @@ -23,7 +23,7 @@ def _check_api(self, *args, **kwargs): resp = self.session.post(*args, **kwargs) if resp.status_code != 200: - raise errors.NSQException(resp.status_code, 'api error') + raise NSQException(resp.status_code, 'api error') return resp.text @@ -37,6 +37,6 @@ def _json_api(self, *args, **kwargs): except: msg = 'api error' - raise errors.NSQException(resp.status_code, msg) + raise NSQException(resp.status_code, msg) return resp.json()['data'] diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 2d24b2d..4383ff5 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -12,16 +12,16 @@ from .message import Message from .httpclient import HTTPClient -HOSTNAME = socket.gethostname() +HOSTNAME = socket.gethostname() SHORTNAME = HOSTNAME.split('.')[0] class Nsqd(HTTPClient): def __init__(self, - address = '127.0.0.1', - tcp_port = 4150, + address = '127.0.0.1', + tcp_port = 4150, http_port = 4151, - timeout = 60.0 + timeout = 60.0 ): if not isinstance(address, (str, unicode)): raise errors.NSQException('address must be a string') @@ -32,10 +32,10 @@ def __init__(self, if not isinstance(http_port, int): raise errors.NSQException('http_port must be a int') - self.address = address - self.tcp_port = tcp_port + self.address = address + self.tcp_port = tcp_port self.http_port = http_port - self.timeout = timeout + self.timeout = timeout self.on_response = blinker.Signal() self.on_error = blinker.Signal() @@ -130,10 +130,10 @@ def _send(self): self._empty_send_queue() def reset(self): - self.ready_count = 0 - self.in_flight = 0 - self._buffer = '' - self._socket = None + self.ready_count = 0 + self.in_flight = 0 + self._buffer = '' + self._socket = None self._on_next_response = None def _readn(self, size): @@ -160,20 +160,20 @@ def _read_response(self): return self._readn(size) def read_response(self): - response = self._read_response() + response = self._read_response() frame, data = nsq.unpack_response(response) if frame not in self._frame_handlers: raise errors.NSQFrameError('unknown frame %s' % frame) - frame_handler = self._frame_handlers[frame] - processed_data = frame_handler(data) + frame_handler = self._frame_handlers[frame] + processed_data = frame_handler(data) self._on_next_response = None return frame, processed_data def handle_response(self, data): - if data == '_heartbeat_': + if data == nsq.HEARTBEAT: self.nop() elif self._on_next_response is not None: @@ -193,7 +193,7 @@ def handle_error(self, data): def handle_message(self, data): self.ready_count -= 1 - self.in_flight += 1 + self.in_flight += 1 message = Message(self, *nsq.unpack_message(data)) self.on_message.send(self, message=message) return message @@ -203,13 +203,13 @@ def listen(self): self.read_response() def identify(self, - short_id = SHORTNAME, - long_id = HOSTNAME, + short_id = SHORTNAME, + long_id = HOSTNAME, heartbeat_interval = None ): self.send(nsq.identify({ - 'short_id': short_id, - 'long_id': long_id, + 'short_id': short_id, + 'long_id': long_id, 'heartbeat_interval': heartbeat_interval })) @@ -275,7 +275,7 @@ def multipublish_http(self, topic, messages): return self._check_api( self.url('mput'), params = {'topic': topic}, - data = '\n'.join(messages) + data = '\n'.join(messages) ) def create_topic(self, topic): diff --git a/gnsq/protocal.py b/gnsq/protocal.py index 466e3eb..aec4878 100644 --- a/gnsq/protocal.py +++ b/gnsq/protocal.py @@ -27,6 +27,7 @@ MAGIC_V2 = ' V2' NEWLINE = '\n' +HEARTBEAT = '_heartbeat_' FRAME_TYPE_RESPONSE = 0 FRAME_TYPE_ERROR = 1 diff --git a/gnsq/reader.py b/gnsq/reader.py index 8f283a0..d5aefa4 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -129,7 +129,7 @@ def smallest_depth(self): if len(self.conns) == 0: return None - stats = self.stats + stats = self.stats depths = [(stats.get(c, {}).get('depth'), c) for c in self.conns] return max(depths)[1] @@ -228,9 +228,9 @@ def handle_requeue(self, conn, message_id, timeout): self.logger.debug(template % (conn, message_id, timeout)) self.on_requeue.send( self, - conn = conn, + conn = conn, message_id = message_id, - timeout = timeout + timeout = timeout ) self.update_ready(conn) From f1855b76a325584bdc9a6f7cebe910cbf580f838 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 16 Jun 2014 15:49:46 -0700 Subject: [PATCH 005/113] Cleanup. --- gnsq/errors.py | 10 +--------- gnsq/lookupd.py | 10 +++++----- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/gnsq/errors.py b/gnsq/errors.py index d1442a4..45edffb 100644 --- a/gnsq/errors.py +++ b/gnsq/errors.py @@ -5,14 +5,6 @@ class NSQException(Exception): pass -class NSQInvalidTopic(NSQException): - pass - - -class NSQInvalidChannel(NSQException): - pass - - class NSQRequeueMessage(NSQException): pass @@ -95,7 +87,7 @@ class NSQTouchFailed(NSQErrorCode): 'E_PUT_FAILED': NSQPutFailed, 'E_PUB_FAILED': NSQPubFailed, 'E_MPUB_FAILED': NSQMPubFailed, - 'E_FINISH_FAILED': NSQFinishFailed, + 'E_FINISH_FAILED': NSQFinishFailed, 'E_FIN_FAILED': NSQFinishFailed, 'E_REQUEUE_FAILED': NSQRequeueFailed, 'E_REQ_FAILED': NSQRequeueFailed, diff --git a/gnsq/lookupd.py b/gnsq/lookupd.py index 02a9dda..c19b6b6 100644 --- a/gnsq/lookupd.py +++ b/gnsq/lookupd.py @@ -13,7 +13,7 @@ def lookup(self, topic): nsq.assert_valid_topic_name(topic) return self._json_api( self.url('lookup'), - params = {'topic': topic} + params={'topic': topic} ) def topics(self): @@ -23,7 +23,7 @@ def channels(self, topic): nsq.assert_valid_topic_name(topic) return self._json_api( self.url('channels'), - params = {'topic': topic} + params={'topic': topic} ) def nodes(self): @@ -33,7 +33,7 @@ def delete_topic(self, topic): nsq.assert_valid_topic_name(topic) return self._json_api( self.url('delete_topic'), - params = {'topic': topic} + params={'topic': topic} ) def delete_channel(self, topic, channel): @@ -41,14 +41,14 @@ def delete_channel(self, topic, channel): nsq.assert_valid_channel_name(channel) return self._json_api( self.url('delete_channel'), - params = {'topic': topic, 'channel': channel} + params={'topic': topic, 'channel': channel} ) def tombstone_topic_producer(self, topic, node): nsq.assert_valid_topic_name(topic) return self._json_api( self.url('tombstone_topic_producer'), - params = {'topic': topic, 'node': node} + params={'topic': topic, 'node': node} ) def ping(self): From 69ad3bf57c3c59759329259d9708976787bfe3b3 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 16 Jun 2014 15:51:23 -0700 Subject: [PATCH 006/113] Factor out streams which act as high-level sockets with plug able transport. --- dev-requirements.txt | 6 + gnsq/nsqd.py | 311 +++++++++++++++++++------------------ gnsq/protocal.py | 11 +- gnsq/reader.py | 148 ++++++++++++------ gnsq/states.py | 5 + gnsq/stream/__init__.py | 7 + gnsq/stream/compression.py | 42 +++++ gnsq/stream/defalte.py | 17 ++ gnsq/stream/snappy.py | 17 ++ gnsq/stream/stream.py | 137 ++++++++++++++++ gnsq/util.py | 11 -- gnsq/version.py | 2 + requirements.txt | 6 - 13 files changed, 497 insertions(+), 223 deletions(-) create mode 100644 dev-requirements.txt create mode 100644 gnsq/states.py create mode 100644 gnsq/stream/__init__.py create mode 100644 gnsq/stream/compression.py create mode 100644 gnsq/stream/defalte.py create mode 100644 gnsq/stream/snappy.py create mode 100644 gnsq/stream/stream.py delete mode 100644 gnsq/util.py create mode 100644 gnsq/version.py delete mode 100644 requirements.txt diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..a9adf18 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,6 @@ +# Install gnsq itself +-e . + +# Install our development requirements +ipython +python-snappy diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 4383ff5..72549e3 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -1,50 +1,74 @@ import blinker -import gevent - from gevent import socket -from gevent.queue import Queue -from gevent.event import AsyncResult -from Queue import Empty + +try: + import simplejson as json +except ImportError: + import json # pyflakes.ignore from . import protocal as nsq from . import errors from .message import Message from .httpclient import HTTPClient +from .states import INIT, CONNECTED, DISCONNECTED +from .stream import Stream +from .version import __version__ HOSTNAME = socket.gethostname() SHORTNAME = HOSTNAME.split('.')[0] +USERAGENT = 'gnsq/{}'.format(__version__) class Nsqd(HTTPClient): - def __init__(self, - address = '127.0.0.1', - tcp_port = 4150, - http_port = 4151, - timeout = 60.0 + def __init__( + self, + address='127.0.0.1', + tcp_port=4150, + http_port=4151, + timeout=60.0, + client_id=SHORTNAME, + hostname=HOSTNAME, + heartbeat_interval=30, + output_buffer_size=16 * 1024, + output_buffer_timeout=250, + tls_v1=False, + tls_options=None, + snappy=False, + deflate=False, + deflate_level=6, + sample_rate=0, + user_agent=USERAGENT, ): - if not isinstance(address, (str, unicode)): - raise errors.NSQException('address must be a string') - - if not isinstance(tcp_port, int): - raise errors.NSQException('tcp_port must be a int') - - if not isinstance(http_port, int): - raise errors.NSQException('http_port must be a int') - self.address = address self.tcp_port = tcp_port self.http_port = http_port self.timeout = timeout + self.client_id = client_id + self.hostname = hostname + self.heartbeat_interval = 1000 * heartbeat_interval + self.output_buffer_size = output_buffer_size + self.output_buffer_timeout = output_buffer_timeout + self.tls_v1 = tls_v1 + self.tls_options = tls_options + self.snappy = snappy + self.deflate = deflate + self.deflate_level = deflate_level + self.sample_rate = sample_rate + self.user_agent = user_agent + + self.state = INIT + self.last_ready = 0 + self.ready_count = 0 + self.in_flight = 0 + self.on_response = blinker.Signal() self.on_error = blinker.Signal() self.on_message = blinker.Signal() self.on_finish = blinker.Signal() self.on_requeue = blinker.Signal() - - self._send_worker = None - self._send_queue = Queue() + self.on_close = blinker.Signal() self._frame_handlers = { nsq.FRAME_TYPE_RESPONSE: self.handle_response, @@ -52,123 +76,54 @@ def __init__(self, nsq.FRAME_TYPE_MESSAGE: self.handle_message } - self.reset() - @property - def is_connected(self): - return self._socket is not None + def connected(self): + return self.state == CONNECTED def connect(self): - if self.is_connected: - raise errors.NSQException('already connected') + if self.state not in (INIT, DISCONNECTED): + return - self.reset() - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(self.timeout) + stream = Stream(self.address, self.tcp_port, self.timeout) try: - s.connect((self.address, self.tcp_port)) + stream.connect((self.address, self.tcp_port)) except socket.error as error: raise errors.NSQSocketError(*error) - self._socket = s - self._send_worker = gevent.spawn(self._send) + self.stream = stream + self.state = CONNECTED self.send(nsq.MAGIC_V2) - def _empty_send_queue(self): - while 1: - try: - data, result = self._send_queue.get_nowait() - except Empty: - return - - result.set_exception(errors.NSQException(-1, 'not connected')) - - def join(self, timeout=None): - if self._send_worker is None: - return - self._send_worker.join(timeout) - - def kill(self): - self._socket = None - - if self._send_worker: - worker, self._send_worker = self._send_worker, None - worker.kill() - - self._empty_send_queue() + def close_stream(self): + self.stream.close() + self.state = DISCONNECTED + self.on_close.send(self) def send(self, data, async=False): - if not self.is_connected: - raise errors.NSQException(-1, 'not connected') - - result = AsyncResult() - self._send_queue.put((data, result)) - - if async: - return result - - result.get() - - def _send(self): - while 1: - data, result = self._send_queue.get() - if not self.is_connected: - result.set_exception(errors.NSQException(-1, 'not connected')) - break - - try: - self._socket.send(data) - result.set(True) - - except socket.error as error: - result.set_exception(errors.NSQSocketError(*error)) - - except Exception as error: - result.set_exception(error) - - self._empty_send_queue() - - def reset(self): - self.ready_count = 0 - self.in_flight = 0 - self._buffer = '' - self._socket = None - self._on_next_response = None - - def _readn(self, size): - while len(self._buffer) < size: - if not self.is_connected: - raise errors.NSQException(-1, 'not connected') - - try: - packet = self._socket.recv(4096) - except socket.error as error: - raise errors.NSQSocketError(*error) - - if not packet: - raise errors.NSQSocketError(-1, 'failed to read %d' % size) - - self._buffer += packet - - data = self._buffer[:size] - self._buffer = self._buffer[size:] - return data + try: + return self.stream.send(data, async) + except Exception: + self.close_stream() + raise def _read_response(self): - size = nsq.unpack_size(self._readn(4)) - return self._readn(size) + try: + size = nsq.unpack_size(self.stream.read(4)) + return self.read(size) + except Exception: + self.close_stream() + raise def read_response(self): response = self._read_response() frame, data = nsq.unpack_response(response) if frame not in self._frame_handlers: - raise errors.NSQFrameError('unknown frame %s' % frame) + raise errors.NSQFrameError('unknown frame {}'.format(frame)) frame_handler = self._frame_handlers[frame] processed_data = frame_handler(data) - self._on_next_response = None return frame, processed_data @@ -176,19 +131,16 @@ def handle_response(self, data): if data == nsq.HEARTBEAT: self.nop() - elif self._on_next_response is not None: - self._on_next_response(self, response=data) - self.on_response.send(self, response=data) return data def handle_error(self, data): error = errors.make_error(data) + self.on_error.send(self, error=error) - if self._on_next_response is not None: - self._on_next_response(self, response=error) + if error.fatal: + self.close_stream() - self.on_error.send(self, error=error) return error def handle_message(self, data): @@ -198,21 +150,81 @@ def handle_message(self, data): self.on_message.send(self, message=message) return message + def finish_inflight(self): + self.in_flight -= 1 + def listen(self): - while self.is_connected: + while self.connected: self.read_response() - def identify(self, - short_id = SHORTNAME, - long_id = HOSTNAME, - heartbeat_interval = None - ): + def upgrade_to_tls(self): + self.stream.upgrade_to_tls(self.tls_options) + + def upgrade_to_snappy(self): + self.stream.upgrade_to_snappy() + + def upgrade_to_defalte(self): + self.stream.upgrade_to_defalte(self.deflate_level) + + def identify(self): self.send(nsq.identify({ - 'short_id': short_id, - 'long_id': long_id, - 'heartbeat_interval': heartbeat_interval + # nsqd <0.2.28 + 'short_id': self.client_id, + 'long_id': self.hostname, + + # nsqd 0.2.28+ + 'client_id': self.client_id, + 'hostname': self.hostname, + + # nsqd 0.2.19+ + 'feature_negotiation': True, + 'heartbeat_interval': self.heartbeat_interval, + + # nsqd 0.2.21+ + 'output_buffer_size': self.output_buffer_size, + 'output_buffer_timeout': self.output_buffer_timeout, + + # nsqd 0.2.22+ + 'tls_v1': self.tls_v1, + + # nsqd 0.2.23+ + 'snappy': self.snappy, + 'deflate': self.deflate, + 'deflate_level': self.deflate_level, + + # nsqd nsqd 0.2.25+ + 'sample_rate': self.sample_rate, + 'user_agent': self.user_agent, })) + frame, data = self.read_response() + + if frame == nsq.FRAME_TYPE_ERROR: + raise data + + if data == 'OK': + return + + try: + data = json.loads(data) + + except ValueError: + self.close_stream() + msg = 'failed to parse IDENTIFY response JSON from nsqd: {!r}' + raise errors.NSQException(msg.format(data)) + + if self.tls_v1 and data.get('tls_v1'): + self.upgrade_to_tls() + + elif self.snappy and data.get('snappy'): + self.upgrade_to_snappy() + + elif self.deflate and data.get('deflate'): + self.deflate_level = data.get('deflate_level', self.deflate_level) + self.upgrade_to_defalte() + + return data + def subscribe(self, topic, channel): self.send(nsq.subscribe(topic, channel)) @@ -223,17 +235,18 @@ def multipublish_tcp(self, topic, messages): self.send(nsq.multipublish(topic, messages)) def ready(self, count): + self.last_ready = count self.ready_count = count self.send(nsq.ready(count)) def finish(self, message_id): self.send(nsq.finish(message_id)) - self.in_flight -= 1 + self.finish_inflight() self.on_finish.send(self, message_id=message_id) def requeue(self, message_id, timeout=0): self.send(nsq.requeue(message_id, timeout)) - self.in_flight -= 1 + self.finish_inflight() self.on_requeue.send(self, message_id=message_id, timeout=timeout) def touch(self, message_id): @@ -247,7 +260,7 @@ def nop(self): @property def base_url(self): - return 'http://%s:%s/' % (self.address, self.http_port) + return 'http://{}:{}/'.format(self.address, self.http_port) def _check_connection(self): if self.http_port: @@ -258,8 +271,8 @@ def publish_http(self, topic, data): nsq.assert_valid_topic_name(topic) return self._check_api( self.url('put'), - params = {'topic': topic}, - data = data + params={'topic': topic}, + data=data ) def multipublish_http(self, topic, messages): @@ -269,27 +282,27 @@ def multipublish_http(self, topic, messages): if '\n' not in message: continue - err = 'newlines are not allowed in http multipublish' - raise errors.NSQException(-1, err) + error = 'newlines are not allowed in http multipublish' + raise errors.NSQException(-1, error) return self._check_api( self.url('mput'), - params = {'topic': topic}, - data = '\n'.join(messages) + params={'topic': topic}, + data='\n'.join(messages) ) def create_topic(self, topic): nsq.assert_valid_topic_name(topic) return self._json_api( self.url('create_topic'), - params = {'topic': topic} + params={'topic': topic} ) def delete_topic(self, topic): nsq.assert_valid_topic_name(topic) return self._json_api( self.url('delete_topic'), - params = {'topic': topic} + params={'topic': topic} ) def create_channel(self, topic, channel): @@ -297,7 +310,7 @@ def create_channel(self, topic, channel): nsq.assert_valid_channel_name(channel) return self._json_api( self.url('create_channel'), - params = {'topic': topic, 'channel': channel} + params={'topic': topic, 'channel': channel} ) def delete_channel(self, topic, channel): @@ -305,14 +318,14 @@ def delete_channel(self, topic, channel): nsq.assert_valid_channel_name(channel) return self._json_api( self.url('delete_channel'), - params = {'topic': topic, 'channel': channel} + params={'topic': topic, 'channel': channel} ) def empty_topic(self, topic): nsq.assert_valid_topic_name(topic) return self._json_api( self.url('empty_topic'), - params = {'topic': topic} + params={'topic': topic} ) def empty_channel(self, topic, channel): @@ -320,7 +333,7 @@ def empty_channel(self, topic, channel): nsq.assert_valid_channel_name(channel) return self._json_api( self.url('empty_channel'), - params = {'topic': topic, 'channel': channel} + params={'topic': topic, 'channel': channel} ) def pause_channel(self, topic, channel): @@ -328,7 +341,7 @@ def pause_channel(self, topic, channel): nsq.assert_valid_channel_name(channel) return self._json_api( self.url('pause_channel'), - params = {'topic': topic, 'channel': channel} + params={'topic': topic, 'channel': channel} ) def unpause_channel(self, topic, channel): @@ -336,7 +349,7 @@ def unpause_channel(self, topic, channel): nsq.assert_valid_channel_name(channel) return self._json_api( self.url('unpause_channel'), - params = {'topic': topic, 'channel': channel} + params={'topic': topic, 'channel': channel} ) def stats(self): @@ -349,13 +362,13 @@ def info(self): return self._json_api(self.url('info')) def publish(self, topic, data): - if self.is_connected: + if self.connected: return self.publish_tcp(topic, data) else: return self.publish_http(topic, data) def multipublish(self, topic, messages): - if self.is_connected: + if self.connected: return self.multipublish_tcp(topic, messages) else: return self.multipublish_http(topic, messages) diff --git a/gnsq/protocal.py b/gnsq/protocal.py index aec4878..c737a48 100644 --- a/gnsq/protocal.py +++ b/gnsq/protocal.py @@ -6,7 +6,6 @@ except ImportError: import json # pyflakes.ignore -from .errors import NSQException, NSQInvalidTopic, NSQInvalidChannel __all__ = [ 'MAGIC_V2', @@ -56,13 +55,13 @@ def valid_channel_name(channel): def assert_valid_topic_name(topic): if valid_topic_name(topic): return - raise NSQInvalidTopic() + raise ValueError('invalid topic name') def assert_valid_channel_name(channel): if valid_channel_name(channel): return - raise NSQInvalidChannel() + raise ValueError('invalid channel name') # @@ -120,10 +119,10 @@ def multipublish(topic_name, messages): def ready(count): if not isinstance(count, int): - raise NSQException('ready count must be an integer') + raise TypeError('ready count must be an integer') if count < 0: - raise NSQException('ready count cannot be negative') + raise ValueError('ready count cannot be negative') return _command('RDY', None, str(count)) @@ -134,7 +133,7 @@ def finish(message_id): def requeue(message_id, timeout=0): if not isinstance(timeout, int): - raise NSQException('requeue timeout must be an integer') + raise TypeError('requeue timeout must be an integer') return _command('REQ', None, message_id, str(timeout)) diff --git a/gnsq/reader.py b/gnsq/reader.py index d5aefa4..66949e6 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -3,9 +3,10 @@ import gevent import blinker +from itertools import cycle + from .lookupd import Lookupd from .nsqd import Nsqd -from .util import assert_list from .errors import ( NSQException, @@ -15,31 +16,50 @@ class Reader(object): - def __init__(self, + def __init__( + self, topic, channel, - nsqd_tcp_addresses = [], - lookupd_http_addresses = [], - async = False, - max_tries = 5, - max_in_flight = 1, - lookupd_poll_interval = 120, - requeue_delay = 0 + nsqd_tcp_addresses=[], + lookupd_http_addresses=[], + async=False, + max_tries=5, + max_in_flight=1, + requeue_delay=0, + lookupd_poll_interval=60, + lookupd_poll_jitter=0.3, + low_rdy_idle_timeout=10, + max_backoff_duration=128, ): - lookupd_http_addresses = assert_list(lookupd_http_addresses) - self.lookupds = [Lookupd(a) for a in lookupd_http_addresses] - self.nsqd_tcp_addresses = assert_list(nsqd_tcp_addresses) + if not nsqd_tcp_addresses and not lookupd_http_addresses: + raise ValueError('must specify at least on nsqd or lookupd') + + if isinstance(nsqd_tcp_addresses, basestring): + self.nsqd_tcp_addresses = [nsqd_tcp_addresses] + elif isinstance(nsqd_tcp_addresses, (list, tuple)): + self.nsqd_tcp_addresses = nsqd_tcp_addresses + else: + raise TypeError('nsqd_tcp_addresses must be a list or tuple') + + if isinstance(lookupd_http_addresses, basestring): + lookupd_http_addresses = [lookupd_http_addresses] + elif isinstance(lookupd_http_addresses, (list, tuple)): + random.shuffle(lookupd_http_addresses) + else: + raise TypeError('lookupd_http_addresses must be a list or tuple') - if not self.nsqd_tcp_addresses and not self.lookupds: - raise NSQException('must specify at least on nsqd or lookupd') + self.lookupds = [Lookupd(a) for a in lookupd_http_addresses] + self.iterlookupds = cycle(self.lookupds) self.topic = topic self.channel = channel self.async = async self.max_tries = max_tries self.max_in_flight = max_in_flight - self.lookupd_poll_interval = lookupd_poll_interval self.requeue_delay = requeue_delay + self.lookupd_poll_interval = lookupd_poll_interval + self.lookupd_poll_jitter = lookupd_poll_jitter + self.logger = logging.getLogger(__name__) self.on_response = blinker.Signal() @@ -47,8 +67,10 @@ def __init__(self, self.on_message = blinker.Signal() self.on_finish = blinker.Signal() self.on_requeue = blinker.Signal() + self.on_exception = blinker.Signal() self.conns = set() + self.pending = set() self.stats = {} def start(self): @@ -56,40 +78,49 @@ def start(self): self.query_lookupd() self.update_stats() self._poll() + # TODO: run _redistribute_rdy_state def connection_max_in_flight(self): return max(1, self.max_in_flight / max(1, len(self.conns))) + def is_starved(self): + for conn in self.conns: + # FIXME + if conn.in_flight > 0 and conn.in_flight >= (conn.last_rdy * 0.85): + return True + return False + def query_nsqd(self): self.logger.debug('querying nsqd...') for address in self.nsqd_tcp_addresses: address, port = address.split(':') - self.connect_to_nsqd(address, int(port)) + conn = Nsqd(address, int(port)) + self.connect_to_nsqd(conn) + + def query_lookupd(self, lookupd): + self.logger.debug('querying lookupd...') + lookupd = self.iterlookupds.next() - def _query_lookupd(self, lookupd): try: producers = lookupd.lookup(self.topic)['producers'] except Exception as error: - template = 'Failed to lookup %s on %s (%s)' - data = (self.topic, lookupd.address, error) - self.logger.warn(template % data) + msg = 'Failed to lookup {} on {} ({})' + self.logger.warn(msg.format(self.topic, lookupd.address, error)) return for producer in producers: - self.connect_to_nsqd( - producer.get('address') or producer['hostname'], + conn = Nsqd( + producer.get('broadcast_address') or producer['address'], producer['tcp_port'], producer['http_port'] ) - - def query_lookupd(self): - self.logger.debug('querying lookupd...') - for lookupd in self.lookupds: - self._query_lookupd(lookupd) + self.connect_to_nsqd(conn) def _poll(self): - gevent.sleep(random.random() * self.lookupd_poll_interval * 0.1) + delay = self.lookupd_poll_interval * self.lookupd_poll_jitter + gevent.sleep(random.random() * delay) + while 1: gevent.sleep(self.lookupd_poll_interval) self.query_nsqd() @@ -107,7 +138,8 @@ def get_stats(self, conn): try: stats = conn.stats() except Exception as error: - self.logger.warn('[%s] stats lookup failed (%r)' % (conn, error)) + msg = '[{}] stats lookup failed ({!r})'.format(conn, error) + self.logger.warn(msg) return None if stats is None: @@ -146,13 +178,16 @@ def publish(self, topic, message): conn.publish(topic, message) - def connect_to_nsqd(self, address, tcp_port, http_port=None): - conn = Nsqd(address, tcp_port, http_port) + def connect_to_nsqd(self, conn): if conn in self.conns: - self.logger.debug('[%s] already connected' % conn) + self.logger.debug('[{}] already connected'.format(conn)) + return + + if conn in self.pending: + self.logger.debug('[{}] already pending'.format(conn)) return - self.logger.debug('[%s] connecting...' % conn) + self.logger.debug('[{}] connecting...'.format(conn)) conn.on_response.connect(self.handle_response) conn.on_error.connect(self.handle_error) @@ -160,42 +195,51 @@ def connect_to_nsqd(self, address, tcp_port, http_port=None): conn.on_finish.connect(self.handle_finish) conn.on_requeue.connect(self.handle_requeue) + self.pending.add(conn) + try: conn.connect() conn.identify() conn.subscribe(self.topic, self.channel) conn.ready(self.connection_max_in_flight()) + except NSQException as error: - self.logger.debug('[%s] connection failed (%r)' % (conn, error)) + msg = '[{}] connection failed ({!r})'.format(conn, error) + self.logger.debug(msg) return - self.logger.info('[%s] connection successful' % conn) + finally: + self.pending.remove(conn) + self.conns.add(conn) conn.worker = gevent.spawn(self._listen, conn) + self.logger.info('[{}] connection successful'.format(conn)) + def _listen(self, conn): try: conn.listen() except NSQException as error: - self.logger.warning('[%s] connection lost (%r)' % (conn, error)) + msg = '[{}] connection lost ({!r})'.format(conn, error) + self.logger.warning(msg) self.conns.remove(conn) - conn.kill() + conn.kill() # FIXME def handle_response(self, conn, response): - self.logger.debug('[%s] response: %s' % (conn, response)) + self.logger.debug('[{}] response: {}'.format(conn, response)) self.on_response.send(self, conn=conn, response=response) def handle_error(self, conn, error): - self.logger.debug('[%s] error: %s' % (conn, error)) + self.logger.debug('[{}] error: {}'.format(conn, error)) self.on_error.send(self, conn=conn, error=error) def handle_message(self, conn, message): - self.logger.debug('[%s] got message: %s' % (conn, message.id)) + self.logger.debug('[{}] got message: {}'.format(conn, message.id)) if self.max_tries and message.attempts > self.max_tries: - template = "giving up on message '%s' after max tries %d" - self.logger.warning(template, message.id, self.max_tries) + msg = "giving up on message '{}' after max tries {}" + self.logger.warning(msg.format(message.id, self.max_tries)) return message.finish() try: @@ -207,9 +251,10 @@ def handle_message(self, conn, message): except NSQRequeueMessage: pass - except Exception: - template = '[%s] caught exception while handling message' - self.logger.exception(template % conn) + except Exception as error: + msg = '[{}] caught exception while handling message'.format(conn) + self.logger.exception(msg) + self.on_exception(self, conn=conn, message=message, error=error) message.requeue(self.requeue_delay) @@ -219,18 +264,18 @@ def update_ready(self, conn): conn.ready(max_in_flight) def handle_finish(self, conn, message_id): - self.logger.debug('[%s] finished message: %s' % (conn, message_id)) + self.logger.debug('[{}] finished message: {}'.format(conn, message_id)) self.on_finish.send(self, conn=conn, message_id=message_id) self.update_ready(conn) def handle_requeue(self, conn, message_id, timeout): - template = '[%s] requeued message: %s (%s)' - self.logger.debug(template % (conn, message_id, timeout)) + msg = '[{}] requeued message: {} ({})' + self.logger.debug(msg.format(conn, message_id, timeout)) self.on_requeue.send( self, - conn = conn, - message_id = message_id, - timeout = timeout + conn=conn, + message_id=message_id, + timeout=timeout ) self.update_ready(conn) @@ -239,5 +284,6 @@ def close(self): conn.close() def join(self, timeout=None, raise_error=False): + # FIXME workers = [c._send_worker for c in self.conns if c._send_worker] gevent.joinall(workers, timeout, raise_error) diff --git a/gnsq/states.py b/gnsq/states.py new file mode 100644 index 0000000..db3eaa8 --- /dev/null +++ b/gnsq/states.py @@ -0,0 +1,5 @@ +"""Connection states.""" + +INIT = 0 +CONNECTED = 1 +DISCONNECTED = 2 diff --git a/gnsq/stream/__init__.py b/gnsq/stream/__init__.py new file mode 100644 index 0000000..0ed74f5 --- /dev/null +++ b/gnsq/stream/__init__.py @@ -0,0 +1,7 @@ +from __future__ import absolute_import +from .stream import Stream + + +__all__ = [ + 'Stream' +] diff --git a/gnsq/stream/compression.py b/gnsq/stream/compression.py new file mode 100644 index 0000000..f0a9670 --- /dev/null +++ b/gnsq/stream/compression.py @@ -0,0 +1,42 @@ +from __future__ import absolute_import + +import socket +import errno + + +class CompressionSocket(object): + def __init__(self, socket): + self._socket = socket + self._bootstrapped = None + + def __getattr__(self, name): + return getattr(self._socket, name) + + def bootstrap(self, data): + if not data: + return + self._bootstrapped = self.decompress(data) + + def compress(self, data): + return data + + def decompress(self, data): + return data + + def recv(self, size): + if self._bootstrapped: + data = self._bootstrapped + self._bootstrapped = None + return data + + chunk = self._socket.recv(size) + if chunk: + uncompressed = self.decompress(chunk) + + if not uncompressed: + raise socket.error(errno.EWOULDBLOCK) + + return uncompressed + + def send(self, data): + self._socket.send(self.compress(data)) diff --git a/gnsq/stream/defalte.py b/gnsq/stream/defalte.py new file mode 100644 index 0000000..8c724dd --- /dev/null +++ b/gnsq/stream/defalte.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import + +import zlib +from .compression import CompressionSocket + + +class DefalteSocket(CompressionSocket): + def __init__(self, socket, level): + self._decompressor = zlib.decompressobj(level) + self._compressor = zlib.compressobj(level) + super(DefalteSocket, self).__init__(socket) + + def compress(self, data): + return self._compressor.compress(data) + + def decompress(self, data): + return self._decompressor.decompress(data) diff --git a/gnsq/stream/snappy.py b/gnsq/stream/snappy.py new file mode 100644 index 0000000..aeeca3e --- /dev/null +++ b/gnsq/stream/snappy.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import + +import snappy +from .compression import CompressionSocket + + +class SnappySocket(CompressionSocket): + def __init__(self, socket): + self._decompressor = snappy.StreamDecompressor() + self._compressor = snappy.StreamCompressor() + super(SnappySocket, self).__init__(socket) + + def compress(self, data): + return self._compressor.add_chunk(data, compress=True) + + def decompress(self, data): + return self._decompressor.decompress(data) diff --git a/gnsq/stream/stream.py b/gnsq/stream/stream.py new file mode 100644 index 0000000..ee65f55 --- /dev/null +++ b/gnsq/stream/stream.py @@ -0,0 +1,137 @@ +from __future__ import absolute_import +from resource import getpagesize + +import gevent +from gevent import socket +from gevent.queue import Queue +from gevent.event import AsyncResult + +try: + import SSLSocket +except ImportError: + SSLSocket = None # pyflakes.ignore + +from gnsq.states import INIT, CONNECTED, DISCONNECTED +from gnsq.errors import NSQSocketError + +try: + from .snappy import SnappySocket +except ImportError: + SnappySocket = None # pyflakes.ignore + +from .defalte import DefalteSocket + + +class Stream(object): + def __init__(self, address, port, timeout, buffer_size=getpagesize()): + self.address = address + self.port = port + self.timeout = timeout + + self.buffer = '' + self.buffer_size = buffer_size + + self.socket = None + self.worker = None + self.queue = Queue() + self.state = INIT + + @property + def connected(self): + return self.state == CONNECTED + + def ensure_connection(self): + if self.connected: + return + raise NSQSocketError(57, 'Socket is not connected') + + def connect(self): + if self.state not in (INIT, DISCONNECTED): + return + + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.settimeout(self.timeout) + self.socket.connect((self.address, self.port)) + + self.state = CONNECTED + self.worker = gevent.spawn(self.send_loop) + + def read(self, size): + while len(self.buffer) < size: + self.ensure_connection() + + try: + packet = self.socket.recv(self.buffer_size) + except socket.error as error: + raise NSQSocketError(*error) + + if not packet: + raise NSQSocketError(-1, 'failed to read {}'.format(size)) + + self.buffer += packet + + data = self.buffer[:size] + self.buffer = self.buffer[size:] + + return data + + def send(self, data, async=False): + self.ensure_connection() + + result = AsyncResult() + self.queue.put((data, result)) + + if async: + return result + + result.get() + + def consume_buffer(self): + data = self.buffer + self.buffer = '' + return data + + def close(self, data): + if not self.connected: + return + + self.state = DISCONNECTED + self.queue.put(StopIteration) + self.socket.close() + + def send_loop(self): + for data, result in self.queue: + if not self.connected: + error = NSQSocketError(57, 'Socket is not connected') + result.set_exception(error) + + try: + self.socket.send(data) + result.set() + + except socket.error as error: + result.set_exception(NSQSocketError(*error)) + + except Exception as error: + result.set_exception(error) + + def upgrade_to_tls(self, options): + if SSLSocket is None: + msg = 'tls_v1 requires Python 2.6+ or Python 2.5 w/ pip install ssl' + raise RuntimeError(msg) + + self.ensure_connection() + self.socket = SSLSocket(self.socket, **options) + + def upgrade_to_snappy(self): + if SnappySocket is None: + raise RuntimeError('snappy requires the python-snappy package') + + self.ensure_connection() + self.socket = SnappySocket(self.socket) + self.socket.bootstrap(self.consume_buffer()) + + def upgrade_to_defalte(self, level): + self.ensure_connection() + self.socket = DefalteSocket(self.socket, level) + self.socket.bootstrap(self.consume_buffer()) diff --git a/gnsq/util.py b/gnsq/util.py deleted file mode 100644 index b0a571c..0000000 --- a/gnsq/util.py +++ /dev/null @@ -1,11 +0,0 @@ -from .errors import NSQException - - -def assert_list(item): - if isinstance(item, basestring): - item = [item] - - elif not isinstance(item, (list, set, tuple)): - raise NSQException('must be a list, set or tuple') - - return item diff --git a/gnsq/version.py b/gnsq/version.py new file mode 100644 index 0000000..0267974 --- /dev/null +++ b/gnsq/version.py @@ -0,0 +1,2 @@ +# also update in setup.py +__version__ = '0.0.1' diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 95994dc..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -blinker==1.3 -gevent==1.0.1 -gnsq==0.0.1 -greenlet==0.4.2 -requests==2.2.1 -wheel==0.22.0 From 1ba7dddc8b1c85a16ac8b0108a3872e8e84107cb Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 16 Jun 2014 15:55:55 -0700 Subject: [PATCH 007/113] Read from stream instead of self. --- gnsq/nsqd.py | 8 ++------ gnsq/stream/stream.py | 8 ++++++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 72549e3..c51d960 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -85,11 +85,7 @@ def connect(self): return stream = Stream(self.address, self.tcp_port, self.timeout) - - try: - stream.connect((self.address, self.tcp_port)) - except socket.error as error: - raise errors.NSQSocketError(*error) + stream.connect() self.stream = stream self.state = CONNECTED @@ -110,7 +106,7 @@ def send(self, data, async=False): def _read_response(self): try: size = nsq.unpack_size(self.stream.read(4)) - return self.read(size) + return self.stream.read(size) except Exception: self.close_stream() raise diff --git a/gnsq/stream/stream.py b/gnsq/stream/stream.py index ee65f55..1712c33 100644 --- a/gnsq/stream/stream.py +++ b/gnsq/stream/stream.py @@ -51,7 +51,11 @@ def connect(self): self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.settimeout(self.timeout) - self.socket.connect((self.address, self.port)) + + try: + self.socket.connect((self.address, self.port)) + except socket.error as error: + raise NSQSocketError(*error) self.state = CONNECTED self.worker = gevent.spawn(self.send_loop) @@ -91,7 +95,7 @@ def consume_buffer(self): self.buffer = '' return data - def close(self, data): + def close(self): if not self.connected: return From d70ccd856bb4ddb061ff608716ef15f778380d62 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 16 Jun 2014 16:19:08 -0700 Subject: [PATCH 008/113] Set correct waits for deflate. --- gnsq/stream/defalte.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gnsq/stream/defalte.py b/gnsq/stream/defalte.py index 8c724dd..f445ae9 100644 --- a/gnsq/stream/defalte.py +++ b/gnsq/stream/defalte.py @@ -6,8 +6,9 @@ class DefalteSocket(CompressionSocket): def __init__(self, socket, level): - self._decompressor = zlib.decompressobj(level) - self._compressor = zlib.compressobj(level) + wbits = -zlib.MAX_WBITS + self._decompressor = zlib.decompressobj(wbits) + self._compressor = zlib.compressobj(level, zlib.DEFLATED, wbits) super(DefalteSocket, self).__init__(socket) def compress(self, data): From ee39841597213f87483383b83d60f7b1de5dd736 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 16 Jun 2014 16:37:36 -0700 Subject: [PATCH 009/113] Cleanup socket errors. --- gnsq/stream/compression.py | 6 +++--- gnsq/stream/stream.py | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/gnsq/stream/compression.py b/gnsq/stream/compression.py index f0a9670..95528a0 100644 --- a/gnsq/stream/compression.py +++ b/gnsq/stream/compression.py @@ -1,7 +1,7 @@ from __future__ import absolute_import -import socket -import errno +from errno import EWOULDBLOCK +from gnsq.errors import NSQSocketError class CompressionSocket(object): @@ -34,7 +34,7 @@ def recv(self, size): uncompressed = self.decompress(chunk) if not uncompressed: - raise socket.error(errno.EWOULDBLOCK) + raise NSQSocketError(EWOULDBLOCK, 'Operation would block') return uncompressed diff --git a/gnsq/stream/stream.py b/gnsq/stream/stream.py index 1712c33..8078ada 100644 --- a/gnsq/stream/stream.py +++ b/gnsq/stream/stream.py @@ -1,5 +1,6 @@ from __future__ import absolute_import from resource import getpagesize +from errno import ENOTCONN import gevent from gevent import socket @@ -43,7 +44,7 @@ def connected(self): def ensure_connection(self): if self.connected: return - raise NSQSocketError(57, 'Socket is not connected') + raise NSQSocketError(ENOTCONN, 'Socket is not connected') def connect(self): if self.state not in (INIT, DISCONNECTED): @@ -70,7 +71,7 @@ def read(self, size): raise NSQSocketError(*error) if not packet: - raise NSQSocketError(-1, 'failed to read {}'.format(size)) + self.close() self.buffer += packet @@ -106,7 +107,7 @@ def close(self): def send_loop(self): for data, result in self.queue: if not self.connected: - error = NSQSocketError(57, 'Socket is not connected') + error = NSQSocketError(ENOTCONN, 'Socket is not connected') result.set_exception(error) try: From c9d640058cebbdd40633d1ff5006e4c0fa171a20 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 16 Jun 2014 17:03:05 -0700 Subject: [PATCH 010/113] Limit options for upgrading to tls. --- gnsq/nsqd.py | 2 +- gnsq/stream/stream.py | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index c51d960..f883e1c 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -154,7 +154,7 @@ def listen(self): self.read_response() def upgrade_to_tls(self): - self.stream.upgrade_to_tls(self.tls_options) + self.stream.upgrade_to_tls(**self.tls_options) def upgrade_to_snappy(self): self.stream.upgrade_to_snappy() diff --git a/gnsq/stream/stream.py b/gnsq/stream/stream.py index 8078ada..9ddcb0d 100644 --- a/gnsq/stream/stream.py +++ b/gnsq/stream/stream.py @@ -8,7 +8,7 @@ from gevent.event import AsyncResult try: - import SSLSocket + from gevent.ssl import SSLSocket, PROTOCOL_TLSv1, CERT_NONE except ImportError: SSLSocket = None # pyflakes.ignore @@ -120,13 +120,26 @@ def send_loop(self): except Exception as error: result.set_exception(error) - def upgrade_to_tls(self, options): + def upgrade_to_tls( + self, + keyfile=None, + certfile=None, + cert_reqs=CERT_NONE, + ca_certs=None + ): if SSLSocket is None: msg = 'tls_v1 requires Python 2.6+ or Python 2.5 w/ pip install ssl' raise RuntimeError(msg) self.ensure_connection() - self.socket = SSLSocket(self.socket, **options) + self.socket = SSLSocket( + self.socket, + keyfile=keyfile, + certfile=certfile, + cert_reqs=cert_reqs, + ca_certs=ca_certs, + ssl_version=PROTOCOL_TLSv1, + ) def upgrade_to_snappy(self): if SnappySocket is None: From da149719b176a93740972c6e48c59dae1eac5efd Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 16 Jun 2014 20:59:29 -0700 Subject: [PATCH 011/113] Setup pytest. --- dev-requirements.txt | 4 +++- setup.py | 28 +++++++++++++++++++++++----- tests/__init__.py | 2 -- tests/test_gnsq.py | 9 ++------- tox.ini | 9 +++++++-- 5 files changed, 35 insertions(+), 17 deletions(-) delete mode 100755 tests/__init__.py diff --git a/dev-requirements.txt b/dev-requirements.txt index a9adf18..7e46cea 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,5 +2,7 @@ -e . # Install our development requirements -ipython +flake8 +pytest python-snappy +tox diff --git a/setup.py b/setup.py index ceb03f9..13ce66f 100755 --- a/setup.py +++ b/setup.py @@ -5,16 +5,33 @@ import sys -try: - from setuptools import setup -except ImportError: - from distutils.core import setup +from setuptools import setup +from setuptools.command.test import test as TestCommand if sys.argv[-1] == 'publish': os.system('python setup.py sdist upload') sys.exit() +class PyTest(TestCommand): + user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = None + + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + #import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(self.pytest_args)# or ['-x', 'tests']) + sys.exit(errno) + + setup( name='gnsq', version='0.0.1', @@ -35,5 +52,6 @@ classifiers=[ 'License :: OSI Approved :: MIT License', ], - test_suite='tests', + tests_require=['pytest'], + cmdclass={'test': PyTest}, ) diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100755 index faa18be..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- diff --git a/tests/test_gnsq.py b/tests/test_gnsq.py index 9599d9b..c17bf12 100755 --- a/tests/test_gnsq.py +++ b/tests/test_gnsq.py @@ -8,12 +8,10 @@ Tests for `gnsq` module. """ -import unittest +# from gnsq import Nsqd -from gnsq import nsq - -class TestGnsq(unittest.TestCase): +class TestGnsq(object): def setUp(self): pass @@ -23,6 +21,3 @@ def test_something(self): def tearDown(self): pass - -if __name__ == '__main__': - unittest.main() diff --git a/tox.ini b/tox.ini index ec49931..871cbe2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,14 @@ [tox] -envlist = py27, py34 +envlist = py26, py27, py34 [testenv] setenv = PYTHONPATH = {toxinidir}:{toxinidir}/gnsq commands = python setup.py test deps = - -r{toxinidir}/requirements.txt + -r{toxinidir}/dev-requirements.txt + +[flake8] +max-line-length = 80 +exclude = tests/* +max-complexity = 10 From bceb760cc965993e99c8e6f0dff83bea8e93b026 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 16 Jun 2014 21:06:56 -0700 Subject: [PATCH 012/113] Call py.tests directly. --- Makefile | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8ef6348..a4ead90 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ lint: flake8 gnsq tests test: - python setup.py test + py.test tests test-all: tox diff --git a/setup.py b/setup.py index 13ce66f..f91d356 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ def finalize_options(self): def run_tests(self): #import here, cause outside the eggs aren't loaded import pytest - errno = pytest.main(self.pytest_args)# or ['-x', 'tests']) + errno = pytest.main(self.pytest_args or 'tests') sys.exit(errno) From 4c7e11027645982534366be96f4d3c7a35227760 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 10:49:09 -0700 Subject: [PATCH 013/113] Backoff and reconnect to connections specified in nsqd_tcp_addresses. --- gnsq/reader.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 66949e6..2e99d69 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -4,9 +4,11 @@ import blinker from itertools import cycle +from collections import defaultdict from .lookupd import Lookupd from .nsqd import Nsqd +from .backofftimer import BackoffTimer from .errors import ( NSQException, @@ -37,16 +39,18 @@ def __init__( if isinstance(nsqd_tcp_addresses, basestring): self.nsqd_tcp_addresses = [nsqd_tcp_addresses] elif isinstance(nsqd_tcp_addresses, (list, tuple)): - self.nsqd_tcp_addresses = nsqd_tcp_addresses + self.nsqd_tcp_addresses = set(nsqd_tcp_addresses) else: - raise TypeError('nsqd_tcp_addresses must be a list or tuple') + raise TypeError('nsqd_tcp_addresses must be a list, set or tuple') if isinstance(lookupd_http_addresses, basestring): lookupd_http_addresses = [lookupd_http_addresses] elif isinstance(lookupd_http_addresses, (list, tuple)): + lookupd_http_addresses = list(lookupd_http_addresses) random.shuffle(lookupd_http_addresses) else: - raise TypeError('lookupd_http_addresses must be a list or tuple') + msg = 'lookupd_http_addresses must be a list, set or tuple' + raise TypeError(msg) self.lookupds = [Lookupd(a) for a in lookupd_http_addresses] self.iterlookupds = cycle(self.lookupds) @@ -59,8 +63,10 @@ def __init__( self.requeue_delay = requeue_delay self.lookupd_poll_interval = lookupd_poll_interval self.lookupd_poll_jitter = lookupd_poll_jitter + self.max_backoff_duration = max_backoff_duration self.logger = logging.getLogger(__name__) + self.backofftimers = defaultdict(self.create_backoff) self.on_response = blinker.Signal() self.on_error = blinker.Signal() @@ -117,6 +123,9 @@ def query_lookupd(self, lookupd): ) self.connect_to_nsqd(conn) + def create_backoff(self): + return BackoffTimer(max_interval=self.max_backoff_duration) + def _poll(self): delay = self.lookupd_poll_interval * self.lookupd_poll_jitter gevent.sleep(random.random() * delay) @@ -206,6 +215,7 @@ def connect_to_nsqd(self, conn): except NSQException as error: msg = '[{}] connection failed ({!r})'.format(conn, error) self.logger.debug(msg) + self.handle_connection_failure(conn) return finally: @@ -215,6 +225,7 @@ def connect_to_nsqd(self, conn): conn.worker = gevent.spawn(self._listen, conn) self.logger.info('[{}] connection successful'.format(conn)) + self.handle_connection_success(conn) def _listen(self, conn): try: @@ -223,8 +234,23 @@ def _listen(self, conn): msg = '[{}] connection lost ({!r})'.format(conn, error) self.logger.warning(msg) - self.conns.remove(conn) - conn.kill() # FIXME + self.handle_connection_failure(conn) + + def handle_connection_failure(self, conn): + self.conns.discard(conn) + conn.close_stream() + + if str(conn) not in self.nsqd_tcp_addresses: + return + + seconds = self.backofftimers[str(conn)].failure().get_interval() + self.logger.debug('[{}] retrying in {}s'.format(conn, seconds)) + gevent.spawn_later(seconds, self.connect_to_nsqd, conn) + + def handle_connection_success(self, conn): + if str(conn) not in self.nsqd_tcp_addresses: + return + self.backofftimers[str(conn)].success() def handle_response(self, conn, response): self.logger.debug('[{}] response: {}'.format(conn, response)) From aa1a3a581a0975d97d1aee31a793750a4f94d0bc Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 11:40:46 -0700 Subject: [PATCH 014/113] Get max_rdy_count from feature negotiation. --- gnsq/nsqd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index f883e1c..d23a4a0 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -62,6 +62,7 @@ def __init__( self.last_ready = 0 self.ready_count = 0 self.in_flight = 0 + self.max_rdy_count = 2500 self.on_response = blinker.Signal() self.on_error = blinker.Signal() @@ -209,6 +210,8 @@ def identify(self): msg = 'failed to parse IDENTIFY response JSON from nsqd: {!r}' raise errors.NSQException(msg.format(data)) + self.max_rdy_count = data.get('max_rdy_count', self.max_rdy_count) + if self.tls_v1 and data.get('tls_v1'): self.upgrade_to_tls() From 6cb843d2789cd8b7210b4288a2cba467cb83c360 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 11:41:44 -0700 Subject: [PATCH 015/113] Remove stats monitoring. --- gnsq/reader.py | 62 +++++++++++++------------------------------------- 1 file changed, 16 insertions(+), 46 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 2e99d69..d68f981 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -77,13 +77,11 @@ def __init__( self.conns = set() self.pending = set() - self.stats = {} def start(self): self.query_nsqd() self.query_lookupd() - self.update_stats() - self._poll() + self._poll() # spawn poll only if we have lookupds # TODO: run _redistribute_rdy_state def connection_max_in_flight(self): @@ -91,7 +89,6 @@ def connection_max_in_flight(self): def is_starved(self): for conn in self.conns: - # FIXME if conn.in_flight > 0 and conn.in_flight >= (conn.last_rdy * 0.85): return True return False @@ -103,7 +100,10 @@ def query_nsqd(self): conn = Nsqd(address, int(port)) self.connect_to_nsqd(conn) - def query_lookupd(self, lookupd): + def query_lookupd(self): + if not self.lookupds: + return + self.logger.debug('querying lookupd...') lookupd = self.iterlookupds.next() @@ -134,46 +134,6 @@ def _poll(self): gevent.sleep(self.lookupd_poll_interval) self.query_nsqd() self.query_lookupd() - self.update_stats() - - def update_stats(self): - stats = {} - for conn in self.conns: - stats[conn] = self.get_stats(conn) - - self.stats = stats - - def get_stats(self, conn): - try: - stats = conn.stats() - except Exception as error: - msg = '[{}] stats lookup failed ({!r})'.format(conn, error) - self.logger.warn(msg) - return None - - if stats is None: - return None - - for topic in stats['topics']: - if topic['topic_name'] != self.topic: - continue - - for channel in topic['channels']: - if channel['channel_name'] != self.channel: - continue - - return channel - - return None - - def smallest_depth(self): - if len(self.conns) == 0: - return None - - stats = self.stats - depths = [(stats.get(c, {}).get('depth'), c) for c in self.conns] - - return max(depths)[1] def random_connection(self): if not self.conns: @@ -209,8 +169,18 @@ def connect_to_nsqd(self, conn): try: conn.connect() conn.identify() + + if conn.max_rdy_count < self.max_in_flight: + self.logger.warning(' '.join([ + '[{}] max RDY count {} < reader max in flight {},', + 'truncation possible' + ]).format(conn, conn.max_rdy_count, self.max_in_flight)) + conn.subscribe(self.topic, self.channel) - conn.ready(self.connection_max_in_flight()) + + # Send RDY 1 if we're the first connection + if not self.conns: + conn.ready(1) except NSQException as error: msg = '[{}] connection failed ({!r})'.format(conn, error) From c4b6aec0087a73cfaf8ba412bb6a8d82bfef117f Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 12:13:18 -0700 Subject: [PATCH 016/113] Fixed closing a reader. --- gnsq/reader.py | 78 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index d68f981..b96bda1 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -78,11 +78,39 @@ def __init__( self.conns = set() self.pending = set() - def start(self): + self.workers = [] + self.conn_workers = {} + self.closed = True + + def start(self, block=True): + self.closed = False self.query_nsqd() - self.query_lookupd() - self._poll() # spawn poll only if we have lookupds - # TODO: run _redistribute_rdy_state + + if self.lookupds: + self.query_lookupd() + self.workers.append(gevent.spawn(self._poll_lookupd)) + + # TODO + # self.workers.append(gevent.spawn(self._redistribute_rdy_state)) + + if block: + self.join() + + def close(self, block=True): + self.closed = True + + for worker in self.workers: + worker.kill(block=block) + + for conn in self.conns: + conn.close_stream() + + if block: + self.join() + + def join(self, timeout=None, raise_error=False): + gevent.joinall(self.workers, timeout, raise_error) + gevent.joinall(self.conn_workers.values(), timeout, raise_error) def connection_max_in_flight(self): return max(1, self.max_in_flight / max(1, len(self.conns))) @@ -101,9 +129,6 @@ def query_nsqd(self): self.connect_to_nsqd(conn) def query_lookupd(self): - if not self.lookupds: - return - self.logger.debug('querying lookupd...') lookupd = self.iterlookupds.next() @@ -126,11 +151,11 @@ def query_lookupd(self): def create_backoff(self): return BackoffTimer(max_interval=self.max_backoff_duration) - def _poll(self): + def _poll_lookupd(self): delay = self.lookupd_poll_interval * self.lookupd_poll_jitter gevent.sleep(random.random() * delay) - while 1: + while True: gevent.sleep(self.lookupd_poll_interval) self.query_nsqd() self.query_lookupd() @@ -148,6 +173,9 @@ def publish(self, topic, message): conn.publish(topic, message) def connect_to_nsqd(self, conn): + if self.closed: + return + if conn in self.conns: self.logger.debug('[{}] already connected'.format(conn)) return @@ -191,8 +219,10 @@ def connect_to_nsqd(self, conn): finally: self.pending.remove(conn) - self.conns.add(conn) - conn.worker = gevent.spawn(self._listen, conn) + # Check if we've closed since we started + if self.closed: + conn.close_stream() + return self.logger.info('[{}] connection successful'.format(conn)) self.handle_connection_success(conn) @@ -206,22 +236,27 @@ def _listen(self, conn): self.handle_connection_failure(conn) + def handle_connection_success(self, conn): + self.conns.add(conn) + self.conn_workers[conn] = gevent.spawn(self._listen, conn) + + if str(conn) not in self.nsqd_tcp_addresses: + return + + self.backofftimers[conn].success() + def handle_connection_failure(self, conn): self.conns.discard(conn) + self.conn_workers.pop(conn, None) conn.close_stream() if str(conn) not in self.nsqd_tcp_addresses: return - seconds = self.backofftimers[str(conn)].failure().get_interval() + seconds = self.backofftimers[conn].failure().get_interval() self.logger.debug('[{}] retrying in {}s'.format(conn, seconds)) gevent.spawn_later(seconds, self.connect_to_nsqd, conn) - def handle_connection_success(self, conn): - if str(conn) not in self.nsqd_tcp_addresses: - return - self.backofftimers[str(conn)].success() - def handle_response(self, conn, response): self.logger.debug('[{}] response: {}'.format(conn, response)) self.on_response.send(self, conn=conn, response=response) @@ -274,12 +309,3 @@ def handle_requeue(self, conn, message_id, timeout): timeout=timeout ) self.update_ready(conn) - - def close(self): - for conn in self.conns: - conn.close() - - def join(self, timeout=None, raise_error=False): - # FIXME - workers = [c._send_worker for c in self.conns if c._send_worker] - gevent.joinall(workers, timeout, raise_error) From a9eea7689b61f88a6adc12b824a4bfb68de09ed4 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 14:53:41 -0700 Subject: [PATCH 017/113] Add back off throttling on message failures. --- gnsq/backofftimer.py | 3 + gnsq/nsqd.py | 9 +- gnsq/reader.py | 202 +++++++++++++++++++++++++++++++++++++------ gnsq/states.py | 4 + 4 files changed, 191 insertions(+), 27 deletions(-) diff --git a/gnsq/backofftimer.py b/gnsq/backofftimer.py index c12a53c..45a1a97 100644 --- a/gnsq/backofftimer.py +++ b/gnsq/backofftimer.py @@ -9,6 +9,9 @@ def __init__(self, ratio=1, max_interval=None, min_interval=None): self.max_interval = max_interval self.min_interval = min_interval + def is_reset(self): + return self.c == 0 + def reset(self): self.c = 0 return self diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index d23a4a0..796d69f 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -1,4 +1,5 @@ import blinker +import time from gevent import socket try: @@ -59,10 +60,12 @@ def __init__( self.user_agent = user_agent self.state = INIT + self.last_ressponse = time.time() + self.last_message = time.time() self.last_ready = 0 self.ready_count = 0 self.in_flight = 0 - self.max_rdy_count = 2500 + self.max_ready_count = 2500 self.on_response = blinker.Signal() self.on_error = blinker.Signal() @@ -115,6 +118,7 @@ def _read_response(self): def read_response(self): response = self._read_response() frame, data = nsq.unpack_response(response) + self.last_response = time.time() if frame not in self._frame_handlers: raise errors.NSQFrameError('unknown frame {}'.format(frame)) @@ -141,6 +145,7 @@ def handle_error(self, data): return error def handle_message(self, data): + self.last_message = time.time() self.ready_count -= 1 self.in_flight += 1 message = Message(self, *nsq.unpack_message(data)) @@ -210,7 +215,7 @@ def identify(self): msg = 'failed to parse IDENTIFY response JSON from nsqd: {!r}' raise errors.NSQException(msg.format(data)) - self.max_rdy_count = data.get('max_rdy_count', self.max_rdy_count) + self.max_ready_count = data.get('max_rdy_count', self.max_ready_count) if self.tls_v1 and data.get('tls_v1'): self.upgrade_to_tls() diff --git a/gnsq/reader.py b/gnsq/reader.py index b96bda1..d11d44e 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -2,6 +2,7 @@ import random import gevent import blinker +import time from itertools import cycle from collections import defaultdict @@ -9,6 +10,7 @@ from .lookupd import Lookupd from .nsqd import Nsqd from .backofftimer import BackoffTimer +from .states import INIT, RUNNING, BACKOFF, THROTTLED, CLOSED from .errors import ( NSQException, @@ -30,7 +32,7 @@ def __init__( requeue_delay=0, lookupd_poll_interval=60, lookupd_poll_jitter=0.3, - low_rdy_idle_timeout=10, + low_ready_idle_timeout=10, max_backoff_duration=128, ): if not nsqd_tcp_addresses and not lookupd_http_addresses: @@ -63,16 +65,23 @@ def __init__( self.requeue_delay = requeue_delay self.lookupd_poll_interval = lookupd_poll_interval self.lookupd_poll_jitter = lookupd_poll_jitter + self.low_ready_idle_timeout = low_ready_idle_timeout self.max_backoff_duration = max_backoff_duration + self.total_ready_count = 0 + self._need_ready_redistributed = False + self.last_random_ready = time.time() + self.logger = logging.getLogger(__name__) self.backofftimers = defaultdict(self.create_backoff) + self.backoff = self.create_backoff() self.on_response = blinker.Signal() self.on_error = blinker.Signal() self.on_message = blinker.Signal() self.on_finish = blinker.Signal() self.on_requeue = blinker.Signal() + self.on_giving_up = blinker.Signal() self.on_exception = blinker.Signal() self.conns = set() @@ -80,24 +89,33 @@ def __init__( self.workers = [] self.conn_workers = {} - self.closed = True + self.state = INIT + + @property + def is_running(self): + self.state in (RUNNING, BACKOFF, THROTTLED) def start(self, block=True): - self.closed = False + if not self.start == INIT: + return + + self.state = RUNNING self.query_nsqd() if self.lookupds: self.query_lookupd() self.workers.append(gevent.spawn(self._poll_lookupd)) - # TODO - # self.workers.append(gevent.spawn(self._redistribute_rdy_state)) + self.workers.append(gevent.spawn(self._poll_ready)) if block: self.join() def close(self, block=True): - self.closed = True + if not self.is_running: + return + + self.state = CLOSED for worker in self.workers: worker.kill(block=block) @@ -112,15 +130,25 @@ def join(self, timeout=None, raise_error=False): gevent.joinall(self.workers, timeout, raise_error) gevent.joinall(self.conn_workers.values(), timeout, raise_error) - def connection_max_in_flight(self): - return max(1, self.max_in_flight / max(1, len(self.conns))) - def is_starved(self): for conn in self.conns: - if conn.in_flight > 0 and conn.in_flight >= (conn.last_rdy * 0.85): + if conn.in_flight >= max(conn.last_ready * 0.85, 1): return True return False + @property + def connection_max_in_flight(self): + return max(1, self.max_in_flight / max(1, len(self.conns))) + + @property + def total_ready_count(self): + return sum(c.ready_count for c in self.conns) + + def send_ready(self, conn, count): + if (self.total_ready_count + count) > self.max_in_flight: + return + conn.ready(count) + def query_nsqd(self): self.logger.debug('querying nsqd...') for address in self.nsqd_tcp_addresses: @@ -157,9 +185,93 @@ def _poll_lookupd(self): while True: gevent.sleep(self.lookupd_poll_interval) - self.query_nsqd() self.query_lookupd() + def _poll_ready(self): + while True: + gevent.sleep(5) + if not self.need_ready_redistributed: + continue + self.redistribute_ready_state() + + @property + def need_ready_redistributed(self): + if self.state == BACKOFF: + return False + + if self._need_ready_redistributed: + return True + + if len(self.conns) > self.max_in_flight: + return True + + if self.state == THROTTLED and len(self.conns) > 1: + return True + + @need_ready_redistributed.setter + def need_ready_redistributed(self, value): + self._need_ready_redistributed = value + + def redistribute_ready_state(self): + self.need_ready_redistributed = False + + # first set RDY 0 to all connections that have not received a message + # within a configurable timeframe (low_ready_idle_timeout). + for conn in self.conns: + if conn.ready_count == 0: + continue + + if (time.time() - conn.last_message) < self.low_ready_idle_timeout: + continue + + msg = '[{}] idle connection, giving up RDY count'.format(conn) + self.logger.info(msg) + + conn.ready(0) + + if self.state == THROTTLED: + max_in_flight = 1 - self.total_ready + else: + max_in_flight = self.max_in_flight - self.total_ready + + if max_in_flight <= 0: + return + + # randomly walk the list of possible connections and send RDY 1 (up to + # our calculate "max_in_flight"). We only need to send RDY 1 because in + # both cases described above your per connection RDY count would never + # be higher. + # + # We also don't attempt to avoid the connections who previously might + # have had RDY 1 because it would be overly complicated and not actually + # worth it (ie. given enough redistribution rounds it doesn't matter). + conns = list(self.conns) + conns = random.sample(conns, min(max_in_flight, len(self.conn))) + + for conn in conns: + self.logger.info('[{}] redistributing RDY'.format(conn)) + self.send_ready(conn, 1) + + def random_ready_conn(self, conn): + # if all connections aren't getting RDY + # occsionally randomize which connection gets RDY + if len(self.conns) <= self.max_in_flight: + return conn + + if (time.time() - self.last_random_ready) < 30: + return conn + + self.last_random_ready = time.time() + return random.choice(c for c in self.conns if not c.ready_count) + + def update_ready(self, conn): + if self.state in (BACKOFF, THROTTLED): + return + + conn = self.random_ready_conn(conn) + if conn.ready_count < max(conn.last_ready * 0.25, 2): + self.send_ready(conn, self.connection_max_in_flight) + def random_connection(self): if not self.conns: return None @@ -173,7 +285,7 @@ def publish(self, topic, message): conn.publish(topic, message) def connect_to_nsqd(self, conn): - if self.closed: + if not self.is_running: return if conn in self.conns: @@ -198,17 +310,17 @@ def connect_to_nsqd(self, conn): conn.connect() conn.identify() - if conn.max_rdy_count < self.max_in_flight: + if conn.max_ready_count < self.max_in_flight: self.logger.warning(' '.join([ '[{}] max RDY count {} < reader max in flight {},', 'truncation possible' - ]).format(conn, conn.max_rdy_count, self.max_in_flight)) + ]).format(conn, conn.max_ready_count, self.max_in_flight)) conn.subscribe(self.topic, self.channel) - # Send RDY 1 if we're the first connection - if not self.conns: - conn.ready(1) + # Send RDY 1 if we're not backing off or we're the first connection + if self.state != BACKOFF or not self.conns: + self.send_ready(conn, 1) except NSQException as error: msg = '[{}] connection failed ({!r})'.format(conn, error) @@ -220,7 +332,7 @@ def connect_to_nsqd(self, conn): self.pending.remove(conn) # Check if we've closed since we started - if self.closed: + if not self.is_running: conn.close_stream() return @@ -250,6 +362,9 @@ def handle_connection_failure(self, conn): self.conn_workers.pop(conn, None) conn.close_stream() + if conn.ready_count: + self.need_ready_redistributed = True + if str(conn) not in self.nsqd_tcp_addresses: return @@ -267,10 +382,12 @@ def handle_error(self, conn, error): def handle_message(self, conn, message): self.logger.debug('[{}] got message: {}'.format(conn, message.id)) + self.update_ready(conn) if self.max_tries and message.attempts > self.max_tries: msg = "giving up on message '{}' after max tries {}" self.logger.warning(msg.format(message.id, self.max_tries)) + self.on_giving_up.send(self, conn, message) return message.finish() try: @@ -289,23 +406,58 @@ def handle_message(self, conn, message): message.requeue(self.requeue_delay) - def update_ready(self, conn): - max_in_flight = self.connection_max_in_flight() - if conn.ready_count < (0.25 * max_in_flight): - conn.ready(max_in_flight) - def handle_finish(self, conn, message_id): self.logger.debug('[{}] finished message: {}'.format(conn, message_id)) + self.backoff.success() + self.handle_backoff() self.on_finish.send(self, conn=conn, message_id=message_id) - self.update_ready(conn) def handle_requeue(self, conn, message_id, timeout): msg = '[{}] requeued message: {} ({})' self.logger.debug(msg.format(conn, message_id, timeout)) + self.backoff.failure() + self.handle_backoff() self.on_requeue.send( self, conn=conn, message_id=message_id, timeout=timeout ) - self.update_ready(conn) + + def handle_backoff(self): + if self.state == BACKOFF: + return + + if self.state == THROTTLED and self.backoff.is_reset(): + return self.complete_backoff() + + if self.backoff.is_reset(): + return + + self.start_backoff() + + def start_backoff(self): + self.state = BACKOFF + + for conn in self.conns: + conn.ready(0) + + interval = self.backoff.get_interval() + self.logger.info('backing off for {} seconds'.format(interval)) + gevent.sleep(interval) + + self.state = THROTTLED + if not self.conns: + return + + conn = self.random_connection() + self.logger.info('[{}] testing backoff state with RDY 1'.format(conn)) + self.send_ready(conn, 1) + + def complete_backoff(self): + self.state = RUNNING + self.logger.info('backoff complete, resuming normal operation') + + count = self.connection_max_in_flight + for conn in self.conns.values(): + self.send_ready(conn, count) diff --git a/gnsq/states.py b/gnsq/states.py index db3eaa8..93ac79a 100644 --- a/gnsq/states.py +++ b/gnsq/states.py @@ -3,3 +3,7 @@ INIT = 0 CONNECTED = 1 DISCONNECTED = 2 +RUNNING = 3 +BACKOFF = 4 +THROTTLED = 5 +CLOSED = 6 From 0b49114a6b0830fa0b05d32803ae52526b8e48ca Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 15:00:19 -0700 Subject: [PATCH 018/113] Return float for interval instead of int. --- gnsq/backofftimer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnsq/backofftimer.py b/gnsq/backofftimer.py index 45a1a97..68f35ec 100644 --- a/gnsq/backofftimer.py +++ b/gnsq/backofftimer.py @@ -1,4 +1,4 @@ -from random import randint +import random class BackoffTimer(object): @@ -26,7 +26,7 @@ def failure(self): def get_interval(self): k = pow(2, self.c) - 1 - interval = randint(0, k) * self.ratio + interval = random.random() * k * self.ratio if self.max_interval is not None: interval = min(interval, self.max_interval) From 0333e74ed667b09f413d0ea4e4594e7573c37cb8 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 15:02:45 -0700 Subject: [PATCH 019/113] Cleanup. --- gnsq/reader.py | 67 +++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index d11d44e..c815f21 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -91,10 +91,6 @@ def __init__( self.conn_workers = {} self.state = INIT - @property - def is_running(self): - self.state in (RUNNING, BACKOFF, THROTTLED) - def start(self, block=True): if not self.start == INIT: return @@ -130,6 +126,11 @@ def join(self, timeout=None, raise_error=False): gevent.joinall(self.workers, timeout, raise_error) gevent.joinall(self.conn_workers.values(), timeout, raise_error) + @property + def is_running(self): + self.state in (RUNNING, BACKOFF, THROTTLED) + + @property def is_starved(self): for conn in self.conns: if conn.in_flight >= max(conn.last_ready * 0.85, 1): @@ -179,6 +180,32 @@ def query_lookupd(self): def create_backoff(self): return BackoffTimer(max_interval=self.max_backoff_duration) + def start_backoff(self): + self.state = BACKOFF + + for conn in self.conns: + conn.ready(0) + + interval = self.backoff.get_interval() + self.logger.info('backing off for {} seconds'.format(interval)) + gevent.sleep(interval) + + self.state = THROTTLED + if not self.conns: + return + + conn = self.random_connection() + self.logger.info('[{}] testing backoff state with RDY 1'.format(conn)) + self.send_ready(conn, 1) + + def complete_backoff(self): + self.state = RUNNING + self.logger.info('backoff complete, resuming normal operation') + + count = self.connection_max_in_flight + for conn in self.conns.values(): + self.send_ready(conn, count) + def _poll_lookupd(self): delay = self.lookupd_poll_interval * self.lookupd_poll_jitter gevent.sleep(random.random() * delay) @@ -431,33 +458,5 @@ def handle_backoff(self): if self.state == THROTTLED and self.backoff.is_reset(): return self.complete_backoff() - if self.backoff.is_reset(): - return - - self.start_backoff() - - def start_backoff(self): - self.state = BACKOFF - - for conn in self.conns: - conn.ready(0) - - interval = self.backoff.get_interval() - self.logger.info('backing off for {} seconds'.format(interval)) - gevent.sleep(interval) - - self.state = THROTTLED - if not self.conns: - return - - conn = self.random_connection() - self.logger.info('[{}] testing backoff state with RDY 1'.format(conn)) - self.send_ready(conn, 1) - - def complete_backoff(self): - self.state = RUNNING - self.logger.info('backoff complete, resuming normal operation') - - count = self.connection_max_in_flight - for conn in self.conns.values(): - self.send_ready(conn, count) + if not self.backoff.is_reset(): + return self.start_backoff() From f0b90d0daaff158d023b103187d1d8e7f0ec5deb Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 15:05:57 -0700 Subject: [PATCH 020/113] Make clear backoffs are for connections. --- gnsq/reader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index c815f21..cac5a35 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -71,9 +71,10 @@ def __init__( self.total_ready_count = 0 self._need_ready_redistributed = False self.last_random_ready = time.time() + self.state = INIT self.logger = logging.getLogger(__name__) - self.backofftimers = defaultdict(self.create_backoff) + self.conn_backoffs = defaultdict(self.create_backoff) self.backoff = self.create_backoff() self.on_response = blinker.Signal() @@ -89,7 +90,6 @@ def __init__( self.workers = [] self.conn_workers = {} - self.state = INIT def start(self, block=True): if not self.start == INIT: @@ -382,7 +382,7 @@ def handle_connection_success(self, conn): if str(conn) not in self.nsqd_tcp_addresses: return - self.backofftimers[conn].success() + self.conn_backoffs[conn].success() def handle_connection_failure(self, conn): self.conns.discard(conn) @@ -395,7 +395,7 @@ def handle_connection_failure(self, conn): if str(conn) not in self.nsqd_tcp_addresses: return - seconds = self.backofftimers[conn].failure().get_interval() + seconds = self.conn_backoffs[conn].failure().get_interval() self.logger.debug('[{}] retrying in {}s'.format(conn, seconds)) gevent.spawn_later(seconds, self.connect_to_nsqd, conn) From 5c37983ff6d67b30cdd943f3ee39e4a2941ce9f4 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 15:41:47 -0700 Subject: [PATCH 021/113] Bugfixes. --- gnsq/nsqd.py | 8 ++++---- gnsq/reader.py | 6 +++--- gnsq/stream/stream.py | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 796d69f..fdabf45 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -81,7 +81,7 @@ def __init__( } @property - def connected(self): + def is_connected(self): return self.state == CONNECTED def connect(self): @@ -156,7 +156,7 @@ def finish_inflight(self): self.in_flight -= 1 def listen(self): - while self.connected: + while self.is_connected: self.read_response() def upgrade_to_tls(self): @@ -366,13 +366,13 @@ def info(self): return self._json_api(self.url('info')) def publish(self, topic, data): - if self.connected: + if self.is_connected: return self.publish_tcp(topic, data) else: return self.publish_http(topic, data) def multipublish(self, topic, messages): - if self.connected: + if self.is_connected: return self.multipublish_tcp(topic, messages) else: return self.multipublish_http(topic, messages) diff --git a/gnsq/reader.py b/gnsq/reader.py index cac5a35..87ce2d6 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -68,7 +68,6 @@ def __init__( self.low_ready_idle_timeout = low_ready_idle_timeout self.max_backoff_duration = max_backoff_duration - self.total_ready_count = 0 self._need_ready_redistributed = False self.last_random_ready = time.time() self.state = INIT @@ -92,7 +91,7 @@ def __init__( self.conn_workers = {} def start(self, block=True): - if not self.start == INIT: + if not self.state == INIT: return self.state = RUNNING @@ -128,7 +127,7 @@ def join(self, timeout=None, raise_error=False): @property def is_running(self): - self.state in (RUNNING, BACKOFF, THROTTLED) + return self.state in (RUNNING, BACKOFF, THROTTLED) @property def is_starved(self): @@ -163,6 +162,7 @@ def query_lookupd(self): try: producers = lookupd.lookup(self.topic)['producers'] + self.logger.debug('found {} producers'.format(len(producers))) except Exception as error: msg = 'Failed to lookup {} on {} ({})' diff --git a/gnsq/stream/stream.py b/gnsq/stream/stream.py index 9ddcb0d..a90770d 100644 --- a/gnsq/stream/stream.py +++ b/gnsq/stream/stream.py @@ -38,11 +38,11 @@ def __init__(self, address, port, timeout, buffer_size=getpagesize()): self.state = INIT @property - def connected(self): + def is_connected(self): return self.state == CONNECTED def ensure_connection(self): - if self.connected: + if self.is_connected: return raise NSQSocketError(ENOTCONN, 'Socket is not connected') @@ -97,7 +97,7 @@ def consume_buffer(self): return data def close(self): - if not self.connected: + if not self.is_connected: return self.state = DISCONNECTED @@ -106,7 +106,7 @@ def close(self): def send_loop(self): for data, result in self.queue: - if not self.connected: + if not self.is_connected: error = NSQSocketError(ENOTCONN, 'Socket is not connected') result.set_exception(error) From aea470a0d6fa2a46395704b7fbb1776c5b6e8cef Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 16:55:44 -0700 Subject: [PATCH 022/113] Add configurable name to readers. --- gnsq/reader.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 87ce2d6..1655c31 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -26,6 +26,7 @@ def __init__( channel, nsqd_tcp_addresses=[], lookupd_http_addresses=[], + name=None, async=False, max_tries=5, max_in_flight=1, @@ -68,11 +69,16 @@ def __init__( self.low_ready_idle_timeout = low_ready_idle_timeout self.max_backoff_duration = max_backoff_duration + if name: + self.name = name + else: + self.name = '{}.{}.{}'.format(__name__, self.topic, self.channel) + self._need_ready_redistributed = False self.last_random_ready = time.time() self.state = INIT - self.logger = logging.getLogger(__name__) + self.logger = logging.getLogger(self.name) self.conn_backoffs = defaultdict(self.create_backoff) self.backoff = self.create_backoff() From a71fe276bc4db6298009dddde635f1b0244b3f87 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 16:56:11 -0700 Subject: [PATCH 023/113] Add basic and command tests. --- tests/test_basic.py | 75 +++++++++++++++++++++++++++++++++++++++++++ tests/test_command.py | 69 +++++++++++++++++++++++++++++++++++++++ tests/test_gnsq.py | 23 ------------- 3 files changed, 144 insertions(+), 23 deletions(-) create mode 100644 tests/test_basic.py create mode 100644 tests/test_command.py delete mode 100755 tests/test_gnsq.py diff --git a/tests/test_basic.py b/tests/test_basic.py new file mode 100644 index 0000000..445536e --- /dev/null +++ b/tests/test_basic.py @@ -0,0 +1,75 @@ + + +from gnsq import BackoffTimer +from gnsq import protocal as nsq + + +TOPICS = [ + ('valid_name', True), + ('invalid name with space', False), + ('invalid_name_due_to_length_this_is_really_really_really_long', False), + ('test-with_period.', True), + ('test#ephemeral', False), + ('test:ephemeral', False), +] + +CHANNELS = [ + ('test', True), + ('test-with_period.', True), + ('test#ephemeral', True), + ('invalid_name_due_to_length_this_is_really_really_really_long', False), + ('invalid name with space', False), +] + + +def pytest_generate_tests(metafunc): + if metafunc.function == test_topic_names: + for name, good in TOPICS: + metafunc.addcall(funcargs=dict(name=name, good=good)) + + if metafunc.function == test_channel_names: + for name, good in CHANNELS: + metafunc.addcall(funcargs=dict(name=name, good=good)) + + +def test_topic_names(name, good): + assert nsq.valid_topic_name(name) == good + + +def test_channel_names(name, good): + assert nsq.valid_channel_name(name) == good + + +def test_backoff_timer(): + timer = BackoffTimer(max_interval=1000) + assert timer.get_interval() == 0 + assert timer.is_reset() + + timer.success() + assert timer.get_interval() == 0 + assert timer.is_reset() + + timer.failure() + assert timer.c == 1 + assert not timer.is_reset() + + for _ in xrange(100): + interval = timer.get_interval() + assert interval > 0 and interval < 2 + + timer.failure() + assert timer.c == 2 + + for _ in xrange(100): + interval = timer.get_interval() + assert interval > 0 and interval < 4 + + timer.success().success() + assert timer.get_interval() == 0 + assert timer.is_reset() + + for _ in xrange(100): + timer.failure() + + assert timer.c == 100 + assert timer.get_interval() == 1000 diff --git a/tests/test_command.py b/tests/test_command.py new file mode 100644 index 0000000..c9dbf37 --- /dev/null +++ b/tests/test_command.py @@ -0,0 +1,69 @@ +import struct +import pytest + +try: + import simplejson as json +except ImportError: + import json # pyflakes.ignore + +from gnsq import protocal as nsq + + +def pytest_generate_tests(metafunc): + identify_dict_ascii = {'a': 1, 'b': 2} + identify_dict_unicode = {'c': u'w\xc3\xa5\xe2\x80\xa0'} + identify_body_ascii = json.dumps(identify_dict_ascii) + identify_body_unicode = json.dumps(identify_dict_unicode) + + msgs = ['asdf', 'ghjk', 'abcd'] + mpub_body = struct.pack('>l', len(msgs)) + ''.join(struct.pack('>l', len(m)) + m for m in msgs) + if metafunc.function == test_command: + for cmd_method, kwargs, result in [ + (nsq.identify, + {'data': identify_dict_ascii}, + 'IDENTIFY\n' + struct.pack('>l', len(identify_body_ascii)) + + identify_body_ascii), + (nsq.identify, + {'data': identify_dict_unicode}, + 'IDENTIFY\n' + struct.pack('>l', len(identify_body_unicode)) + + identify_body_unicode), + (nsq.subscribe, + {'topic_name': 'test_topic', 'channel_name': 'test_channel'}, + 'SUB test_topic test_channel\n'), + (nsq.finish, + {'message_id': 'test'}, + 'FIN test\n'), + (nsq.finish, + {'message_id': u'\u2020est \xfcn\xee\xe7\xf8\u2202\xe9'}, + 'FIN \xe2\x80\xa0est \xc3\xbcn\xc3\xae\xc3\xa7\xc3\xb8\xe2\x88\x82\xc3\xa9\n'), + (nsq.requeue, + {'message_id': 'test'}, + 'REQ test 0\n'), + (nsq.requeue, + {'message_id': 'test', 'timeout': 60}, + 'REQ test 60\n'), + (nsq.touch, + {'message_id': 'test'}, + 'TOUCH test\n'), + (nsq.ready, + {'count': 100}, + 'RDY 100\n'), + (nsq.nop, + {}, + 'NOP\n'), + (nsq.publish, + {'topic_name': 'test', 'data': msgs[0]}, + 'PUB test\n' + struct.pack('>l', len(msgs[0])) + msgs[0]), + (nsq.multipublish, + {'topic_name': 'test', 'messages': msgs}, + 'MPUB test\n' + struct.pack('>l', len(mpub_body)) + mpub_body) + ]: + metafunc.addcall(funcargs=dict(cmd_method=cmd_method, kwargs=kwargs, result=result)) + + +def test_command(cmd_method, kwargs, result): + assert cmd_method(**kwargs) == result + + +def test_unicode_body(): + pytest.raises(TypeError, nsq.publish, 'topic', u'unicode body') diff --git a/tests/test_gnsq.py b/tests/test_gnsq.py deleted file mode 100755 index c17bf12..0000000 --- a/tests/test_gnsq.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -test_gnsq ----------------------------------- - -Tests for `gnsq` module. -""" - -# from gnsq import Nsqd - - -class TestGnsq(object): - - def setUp(self): - pass - - def test_something(self): - pass - - def tearDown(self): - pass From 61fd86817b8a183359b9b86252f52b8f7256a8e4 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 17 Jun 2014 16:57:55 -0700 Subject: [PATCH 024/113] Fix mpub protocol implementation. --- gnsq/protocal.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/gnsq/protocal.py b/gnsq/protocal.py index c737a48..afc8f33 100644 --- a/gnsq/protocal.py +++ b/gnsq/protocal.py @@ -86,13 +86,26 @@ def unpack_message(data): # # Commands # +def _packsize(data): + return struct.pack('>l', len(data)) + + def _packbody(body): if body is None: return '' - return struct.pack('>l', len(body)) + body + if not isinstance(body, str): + raise TypeError('message body must be a byte string') + return _packsize(body) + body + + +def _encode_param(data): + if not isinstance(data, unicode): + return data + return data.encode('utf-8') def _command(cmd, body, *params): + params = tuple(_encode_param(p) for p in params) return ''.join((' '.join((cmd,) + params), NEWLINE, _packbody(body))) @@ -114,7 +127,7 @@ def publish(topic_name, data): def multipublish(topic_name, messages): assert_valid_topic_name(topic_name) data = ''.join(_packbody(m) for m in messages) - return _command('MPUB', data, topic_name) + return _command('MPUB', _packsize(messages) + data, topic_name) def ready(count): From b03b1b7016445f116bebcf0051ca4c58f18389c1 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Thu, 26 Jun 2014 23:40:01 -0700 Subject: [PATCH 025/113] Parametrize basic tests. --- tests/test_basic.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/tests/test_basic.py b/tests/test_basic.py index 445536e..1caef8c 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -1,41 +1,28 @@ - +import pytest from gnsq import BackoffTimer from gnsq import protocal as nsq -TOPICS = [ +@pytest.mark.parametrize('name,good', [ ('valid_name', True), ('invalid name with space', False), ('invalid_name_due_to_length_this_is_really_really_really_long', False), ('test-with_period.', True), ('test#ephemeral', False), ('test:ephemeral', False), -] +]) +def test_topic_names(name, good): + assert nsq.valid_topic_name(name) == good + -CHANNELS = [ +@pytest.mark.parametrize('name,good', [ ('test', True), ('test-with_period.', True), ('test#ephemeral', True), ('invalid_name_due_to_length_this_is_really_really_really_long', False), ('invalid name with space', False), -] - - -def pytest_generate_tests(metafunc): - if metafunc.function == test_topic_names: - for name, good in TOPICS: - metafunc.addcall(funcargs=dict(name=name, good=good)) - - if metafunc.function == test_channel_names: - for name, good in CHANNELS: - metafunc.addcall(funcargs=dict(name=name, good=good)) - - -def test_topic_names(name, good): - assert nsq.valid_topic_name(name) == good - - +]) def test_channel_names(name, good): assert nsq.valid_channel_name(name) == good From 769141143a46e08fdcc6484658193a725c9d01fe Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Thu, 26 Jun 2014 23:40:16 -0700 Subject: [PATCH 026/113] Add nsqd tests. --- tests/test_nsqd.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/test_nsqd.py diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py new file mode 100644 index 0000000..c918519 --- /dev/null +++ b/tests/test_nsqd.py @@ -0,0 +1,63 @@ +from __future__ import with_statement +import struct + +from gevent.event import AsyncResult +from gevent.server import StreamServer + +from gnsq import Nsqd, states + + +class nsqd_handler(object): + def __init__(self, handler): + self.handler = handler + self.result = AsyncResult() + self.server = StreamServer(('127.0.0.1', 0), self) + + def __call__(self, socket, address): + try: + self.handler(socket, address) + self.result.set() + except Exception as error: + self.result.set_exception(error) + + def __enter__(self): + self.server.start() + return self.server + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None: + self.result.get() + self.server.stop() + + +def test_connection(): + + @nsqd_handler + def handle(socket, address): + assert socket.recv(4) == ' V2' + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + assert conn.state == states.INIT + + conn.connect() + assert conn.state == states.CONNECTED + + conn.close_stream() + assert conn.state == states.DISCONNECTED + + +def test_read(): + body = 'hello world' + + @nsqd_handler + def handle(socket, address): + socket.send(struct.pack('>l', len(body))) + socket.send(body) + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + conn.connect() + + assert conn._read_response() == body + conn.close_stream() From 02e565286507e616be3a9e192ee46c7800449936 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Thu, 26 Jun 2014 23:52:03 -0700 Subject: [PATCH 027/113] Parametrize test_read. --- tests/test_nsqd.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index c918519..809e1ae 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -1,5 +1,7 @@ from __future__ import with_statement + import struct +import pytest from gevent.event import AsyncResult from gevent.server import StreamServer @@ -35,6 +37,7 @@ def test_connection(): @nsqd_handler def handle(socket, address): assert socket.recv(4) == ' V2' + assert socket.recv(1) == '' with handle as server: conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) @@ -47,8 +50,12 @@ def handle(socket, address): assert conn.state == states.DISCONNECTED -def test_read(): - body = 'hello world' +@pytest.mark.parametrize('body', [ + 'hello world', + '', + '{"some": "json data"}', +]) +def test_read(body): @nsqd_handler def handle(socket, address): From 663741a7a5d606256150fefb7fc12c44f00e35c7 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 28 Jun 2014 13:21:13 -0700 Subject: [PATCH 028/113] Test nsqd identify, negotiation, commands and messages. --- tests/test_nsqd.py | 142 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 3 deletions(-) diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index 809e1ae..56f80ad 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -1,12 +1,29 @@ from __future__ import with_statement import struct +import json import pytest from gevent.event import AsyncResult from gevent.server import StreamServer -from gnsq import Nsqd, states +from gnsq import Nsqd, Message, states +from gnsq import protocal as nsq + + +def mock_response(frame_type, data): + body_size = 4 + len(data) + body_size_packed = struct.pack('>l', body_size) + frame_type_packed = struct.pack('>l', frame_type) + return body_size_packed + frame_type_packed + data + + +def mock_response_message(timestamp, attempts, id, body): + timestamp_packed = struct.pack('>q', timestamp) + attempts_packed = struct.pack('>h', attempts) + id = "%016d" % id + data = timestamp_packed + attempts_packed + id + body + return mock_response(nsq.FRAME_TYPE_MESSAGE, data) class nsqd_handler(object): @@ -17,10 +34,11 @@ def __init__(self, handler): def __call__(self, socket, address): try: - self.handler(socket, address) - self.result.set() + self.result.set(self.handler(socket, address)) except Exception as error: self.result.set_exception(error) + finally: + socket.close() def __enter__(self): self.server.start() @@ -68,3 +86,121 @@ def handle(socket, address): assert conn._read_response() == body conn.close_stream() + + +def test_identify(): + + @nsqd_handler + def handle(socket, address): + assert socket.recv(4) == ' V2' + assert socket.recv(9) == 'IDENTIFY\n' + + size = nsq.unpack_size(socket.recv(4)) + data = json.loads(socket.recv(size)) + + assert 'gnsq' in data['user_agent'] + socket.send(mock_response(nsq.FRAME_TYPE_RESPONSE, 'OK')) + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + conn.connect() + + assert conn.identify() is None + + +def test_negotiation(): + + @nsqd_handler + def handle(socket, address): + assert socket.recv(4) == ' V2' + assert socket.recv(9) == 'IDENTIFY\n' + + size = nsq.unpack_size(socket.recv(4)) + data = json.loads(socket.recv(size)) + + assert 'gnsq' in data['user_agent'] + resp = json.dumps({'test': 42}) + socket.send(mock_response(nsq.FRAME_TYPE_RESPONSE, resp)) + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + conn.connect() + + assert conn.identify()['test'] == 42 + + +@pytest.mark.parametrize('command,args,resp', [ + ('subscribe', ('topic', 'channel'), 'SUB topic channel\n'), + ('subscribe', ('foo', 'bar'), 'SUB foo bar\n'), + ('ready', (0,), 'RDY 0\n'), + ('ready', (1,), 'RDY 1\n'), + ('ready', (42,), 'RDY 42\n'), + ('finish', ('0000000000000000',), 'FIN 0000000000000000\n'), + ('finish', ('deadbeafdeadbeaf',), 'FIN deadbeafdeadbeaf\n'), + ('requeue', ('0000000000000000',), 'REQ 0000000000000000 0\n'), + ('requeue', ('deadbeafdeadbeaf', 0), 'REQ deadbeafdeadbeaf 0\n'), + ('requeue', ('deadbeafdeadbeaf', 42), 'REQ deadbeafdeadbeaf 42\n'), + ('touch', ('0000000000000000',), 'TOUCH 0000000000000000\n'), + ('touch', ('deadbeafdeadbeaf',), 'TOUCH deadbeafdeadbeaf\n'), + ('close', (), 'CLS\n'), + ('nop', (), 'NOP\n'), +]) +def test_command(command, args, resp): + + @nsqd_handler + def handle(socket, address): + assert socket.recv(4) == ' V2' + assert socket.recv(len(resp)) == resp + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + conn.connect() + getattr(conn, command)(*args) + + +def test_sync_receive_messages(): + + @nsqd_handler + def handle(socket, address): + assert socket.recv(4) == ' V2' + assert socket.recv(9) == 'IDENTIFY\n' + + size = nsq.unpack_size(socket.recv(4)) + data = json.loads(socket.recv(size)) + + assert isinstance(data, dict) + socket.send(mock_response(nsq.FRAME_TYPE_RESPONSE, 'OK')) + + msg = 'SUB topic channel\n' + assert socket.recv(len(msg)) == msg + socket.send(mock_response(nsq.FRAME_TYPE_RESPONSE, 'OK')) + + for i in xrange(10): + assert socket.recv(6) == 'RDY 1\n' + + body = json.dumps({'data': {'test_key': i}}) + ts = i * 1000 * 1000 + socket.send(mock_response_message(ts, i, i, body)) + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + conn.connect() + + assert conn.identify() is None + + conn.subscribe('topic', 'channel') + frame, data = conn.read_response() + + assert frame == nsq.FRAME_TYPE_RESPONSE + assert data == 'OK' + + for i in xrange(10): + conn.ready(1) + frame, msg = conn.read_response() + + assert frame == nsq.FRAME_TYPE_MESSAGE + assert isinstance(msg, Message) + assert msg.timestamp == i * 1000 * 1000 + assert msg.id == '%016d' % i + assert msg.attempts == i + assert json.loads(msg.body)['data']['test_key'] == i From 74ae44b23418573136b7552f394f2476c1ae081f Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 29 Jun 2014 20:48:59 -0700 Subject: [PATCH 029/113] Conform to cookiecutter-pypackage. --- AUTHORS.rst | 13 + CHANGELOG.md | 5 - CONTRIBUTING.rst | 111 ++++++ HISTORY.rst | 9 + LICENSE | 25 +- MANIFEST.in | 8 +- Makefile | 6 +- README.md | 12 - README.rst | 23 ++ docs/Makefile | 2 +- docs/authors.rst | 1 + docs/conf.py | 10 +- docs/contributing.rst | 1 + docs/history.rst | 1 + docs/index.rst | 5 +- docs/installation.rst | 2 +- docs/make.bat | 484 +++++++++++------------ docs/readme.rst | 1 + docs/usage.rst | 2 +- gnsq/__init__.py | 6 + dev-requirements.txt => requirements.txt | 1 + setup.cfg | 2 +- setup.py | 68 ++-- tox.ini | 6 +- 24 files changed, 471 insertions(+), 333 deletions(-) create mode 100644 AUTHORS.rst delete mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.rst create mode 100644 HISTORY.rst delete mode 100644 README.md create mode 100644 README.rst create mode 100644 docs/authors.rst create mode 100644 docs/contributing.rst create mode 100644 docs/history.rst create mode 100644 docs/readme.rst rename dev-requirements.txt => requirements.txt (89%) diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..8d64416 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,13 @@ +======= +Credits +======= + +Development Lead +---------------- + +* Trevor Olson + +Contributors +------------ + +None yet. Why not be the first? \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 5d7dea7..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,5 +0,0 @@ -# Changelog - -## 0.0.1 - - -Initial release. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..5d58c78 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,111 @@ +============ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every +little bit helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +Report Bugs +~~~~~~~~~~~ + +Report bugs at https://github.com/wtolson/gnsq/issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +Fix Bugs +~~~~~~~~ + +Look through the GitHub issues for bugs. Anything tagged with "bug" +is open to whoever wants to implement it. + +Implement Features +~~~~~~~~~~~~~~~~~~ + +Look through the GitHub issues for features. Anything tagged with "feature" +is open to whoever wants to implement it. + +Write Documentation +~~~~~~~~~~~~~~~~~~~ + +gnsq could always use more documentation, whether as part of the +official gnsq docs, in docstrings, or even on the web in blog posts, +articles, and such. + +Submit Feedback +~~~~~~~~~~~~~~~ + +The best way to send feedback is to file an issue at https://github.com/wtolson/gnsq/issues. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up `gnsq` for local development. + +1. Fork the `gnsq` repo on GitHub. +2. Clone your fork locally:: + + $ git clone git@github.com:your_name_here/gnsq.git + +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: + + $ mkvirtualenv gnsq + $ cd gnsq/ + $ python setup.py develop + +4. Create a branch for local development:: + + $ git checkout -b name-of-your-bugfix-or-feature + + Now you can make your changes locally. + +5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: + + $ flake8 gnsq tests + $ python setup.py test + $ tox + + To get flake8 and tox, just pip install them into your virtualenv. + +6. Commit your changes and push your branch to GitHub:: + + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + +7. Submit a pull request through the GitHub website. + +Pull Request Guidelines +----------------------- + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 2.6, 2.7, and 3.3, 3.4, and for PyPy. Check + https://travis-ci.org/wtolson/gnsq/pull_requests + and make sure that the tests pass for all supported Python versions. + +Tips +---- + +To run a subset of tests:: + + $ python -m unittest tests.test_gnsq \ No newline at end of file diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..5f35edc --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,9 @@ +.. :changelog: + +History +------- + +0.1.0 (2014-01-11) +--------------------- + +* First release on PyPI. \ No newline at end of file diff --git a/LICENSE b/LICENSE index 89de354..7a983bd 100644 --- a/LICENSE +++ b/LICENSE @@ -1,17 +1,12 @@ -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Copyright (c) 2014, Trevor Olson +All rights reserved. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +* Neither the name of gnsq nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index edbfa1e..6fd9409 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,11 @@ -include CHANGELOG.md +include AUTHORS.rst +include CONTRIBUTING.rst +include HISTORY.rst include LICENSE -include README.md +include README.rst recursive-include tests * recursive-exclude * __pycache__ recursive-exclude * *.py[co] -recursive-include docs *.rst conf.py Makefile make.bat +recursive-include docs *.rst conf.py Makefile make.bat \ No newline at end of file diff --git a/Makefile b/Makefile index a4ead90..e19ad98 100644 --- a/Makefile +++ b/Makefile @@ -28,13 +28,13 @@ lint: flake8 gnsq tests test: - py.test tests + py.test test-all: tox coverage: - coverage run --source gnsq setup.py test + coverage run --source gnsq py.test coverage report -m coverage html open htmlcov/index.html @@ -54,4 +54,4 @@ release: clean dist: clean python setup.py sdist python setup.py bdist_wheel - ls -l dist + ls -l dist \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index 80aa98f..0000000 --- a/README.md +++ /dev/null @@ -1,12 +0,0 @@ -gnsq -==== - -A gevent based NSQ driver for Python. - -* Free software: MIT license -* Documentation: http://gnsq.rtfd.org. - -Features --------- - -* TODO diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..a17c675 --- /dev/null +++ b/README.rst @@ -0,0 +1,23 @@ +=============================== +gnsq +=============================== + +.. image:: https://badge.fury.io/py/gnsq.svg + :target: http://badge.fury.io/py/gnsq + +.. image:: https://travis-ci.org/wtolson/gnsq.svg?branch=master + :target: https://travis-ci.org/wtolson/gnsq + +.. image:: https://pypip.in/d/gnsq/badge.png + :target: https://pypi.python.org/pypi/gnsq + + +A gevent based NSQ driver for Python. + +* Free software: BSD license +* Documentation: http://gnsq.readthedocs.org. + +Features +-------- + +* TODO \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile index 0e35bee..1005e26 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -174,4 +174,4 @@ xml: pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." \ No newline at end of file diff --git a/docs/authors.rst b/docs/authors.rst new file mode 100644 index 0000000..94292d0 --- /dev/null +++ b/docs/authors.rst @@ -0,0 +1 @@ +.. include:: ../AUTHORS.rst \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index ef0540f..58e2678 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,7 +56,7 @@ # General information about the project. project = u'gnsq' -copyright = u'2014, William Trevor Olson' +copyright = u'2014, Trevor Olson' # The version info for the project you're documenting, acts as replacement # for |version| and |release|, also used in various other places throughout @@ -210,7 +210,7 @@ latex_documents = [ ('index', 'gnsq.tex', u'gnsq Documentation', - u'William Trevor Olson', 'manual'), + u'Trevor Olson', 'manual'), ] # The name of an image file (relative to this directory) to place at @@ -241,7 +241,7 @@ man_pages = [ ('index', 'gnsq', u'gnsq Documentation', - [u'William Trevor Olson'], 1) + [u'Trevor Olson'], 1) ] # If true, show URL addresses after external links. @@ -256,7 +256,7 @@ texinfo_documents = [ ('index', 'gnsq', u'gnsq Documentation', - u'William Trevor Olson', + u'Trevor Olson', 'gnsq', 'One line description of project.', 'Miscellaneous'), @@ -272,4 +272,4 @@ #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +#texinfo_no_detailmenu = False \ No newline at end of file diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..3bdd7dc --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst \ No newline at end of file diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 0000000..bec23d8 --- /dev/null +++ b/docs/history.rst @@ -0,0 +1 @@ +.. include:: ../HISTORY.rst \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 2711ce0..d104dae 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,8 +11,12 @@ Contents: .. toctree:: :maxdepth: 2 + readme installation usage + contributing + authors + history Indices and tables ================== @@ -20,4 +24,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/docs/installation.rst b/docs/installation.rst index 90f1c48..64d8c79 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -9,4 +9,4 @@ At the command line:: Or, if you have virtualenvwrapper installed:: $ mkvirtualenv gnsq - $ pip install gnsq + $ pip install gnsq \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat index fec43bb..2b44764 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,242 +1,242 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\complexity.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\complexity.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\complexity.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\complexity.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end \ No newline at end of file diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 0000000..6b2b3ec --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1 @@ +.. include:: ../README.rst \ No newline at end of file diff --git a/docs/usage.rst b/docs/usage.rst index e524b38..33755bf 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -4,4 +4,4 @@ Usage To use gnsq in a project:: - import gnsq + import gnsq \ No newline at end of file diff --git a/gnsq/__init__.py b/gnsq/__init__.py index f558a84..b1bb16d 100644 --- a/gnsq/__init__.py +++ b/gnsq/__init__.py @@ -1,9 +1,15 @@ +# -*- coding: utf-8 -*- + from .reader import Reader from .nsqd import Nsqd from .lookupd import Lookupd from .message import Message from .backofftimer import BackoffTimer +__author__ = 'Trevor Olson' +__email__ = 'trevor@heytrevor.com' +__version__ = '0.1.0' + __all__ = [ 'Reader', 'Nsqd', diff --git a/dev-requirements.txt b/requirements.txt similarity index 89% rename from dev-requirements.txt rename to requirements.txt index 7e46cea..a0c8ef3 100644 --- a/dev-requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ flake8 pytest python-snappy tox +wheel>=0.22 diff --git a/setup.cfg b/setup.cfg index 5e40900..0a8df87 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [wheel] -universal = 1 +universal = 1 \ No newline at end of file diff --git a/setup.py b/setup.py index f91d356..a4914fb 100755 --- a/setup.py +++ b/setup.py @@ -1,57 +1,45 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import os -import sys +try: + from setuptools import setup +except ImportError: + from distutils.core import setup -from setuptools import setup -from setuptools.command.test import test as TestCommand +readme = open('README.rst').read() +history = open('HISTORY.rst').read().replace('.. :changelog:', '') -if sys.argv[-1] == 'publish': - os.system('python setup.py sdist upload') - sys.exit() - - -class PyTest(TestCommand): - user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] - - def initialize_options(self): - TestCommand.initialize_options(self) - self.pytest_args = None - - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True - - def run_tests(self): - #import here, cause outside the eggs aren't loaded - import pytest - errno = pytest.main(self.pytest_args or 'tests') - sys.exit(errno) +requirements = [ + 'gevent', + 'blinker', + 'requests', +] setup( name='gnsq', - version='0.0.1', + version='0.1.0', description='A gevent based NSQ driver for Python.', - long_description=open('README.md').read(), - author='William Trevor Olson', + long_description=readme + '\n\n' + history, + author='Trevor Olson', author_email='trevor@heytrevor.com', url='https://github.com/wtolson/gnsq', - packages=['gnsq'], - include_package_data=True, - install_requires=[ - 'gevent', - 'blinker', - 'requests', + packages=[ + 'gnsq', ], - license="MIT", + package_dir={'gnsq': + 'gnsq'}, + include_package_data=True, + install_requires=requirements, + license="BSD", zip_safe=False, + keywords='gnsq', classifiers=[ - 'License :: OSI Approved :: MIT License', - ], - tests_require=['pytest'], - cmdclass={'test': PyTest}, + 'Development Status :: 2 - Pre-Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Natural Language :: English', + 'Programming Language :: Python :: 2.7', + ] ) diff --git a/tox.ini b/tox.ini index 871cbe2..66f014c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,14 @@ [tox] -envlist = py26, py27, py34 +envlist = py26, py27, py33, py34 [testenv] setenv = PYTHONPATH = {toxinidir}:{toxinidir}/gnsq commands = python setup.py test deps = - -r{toxinidir}/dev-requirements.txt + -r{toxinidir}/requirements.txt [flake8] max-line-length = 80 exclude = tests/* -max-complexity = 10 +max-complexity = 10 \ No newline at end of file From 88a4ce2a17d8e07f79c68468f7e1bae69252b2b2 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 29 Jun 2014 21:07:04 -0700 Subject: [PATCH 030/113] Add tests for nsqd errors. --- tests/mock_server.py | 26 ++++++++++++++++ tests/test_nsqd.py | 74 ++++++++++++++++++++++++-------------------- 2 files changed, 66 insertions(+), 34 deletions(-) create mode 100644 tests/mock_server.py diff --git a/tests/mock_server.py b/tests/mock_server.py new file mode 100644 index 0000000..ec70ab8 --- /dev/null +++ b/tests/mock_server.py @@ -0,0 +1,26 @@ +from gevent.event import AsyncResult +from gevent.server import StreamServer + + +class mock_server(object): + def __init__(self, handler): + self.handler = handler + self.result = AsyncResult() + self.server = StreamServer(('127.0.0.1', 0), self) + + def __call__(self, socket, address): + try: + self.result.set(self.handler(socket, address)) + except Exception as error: + self.result.set_exception(error) + finally: + socket.close() + + def __enter__(self): + self.server.start() + return self.server + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None: + self.result.get() + self.server.stop() diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index 56f80ad..b38411d 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -4,12 +4,11 @@ import json import pytest -from gevent.event import AsyncResult -from gevent.server import StreamServer - -from gnsq import Nsqd, Message, states +from gnsq import Nsqd, Message, states, errors from gnsq import protocal as nsq +from mock_server import mock_server + def mock_response(frame_type, data): body_size = 4 + len(data) @@ -26,33 +25,9 @@ def mock_response_message(timestamp, attempts, id, body): return mock_response(nsq.FRAME_TYPE_MESSAGE, data) -class nsqd_handler(object): - def __init__(self, handler): - self.handler = handler - self.result = AsyncResult() - self.server = StreamServer(('127.0.0.1', 0), self) - - def __call__(self, socket, address): - try: - self.result.set(self.handler(socket, address)) - except Exception as error: - self.result.set_exception(error) - finally: - socket.close() - - def __enter__(self): - self.server.start() - return self.server - - def __exit__(self, exc_type, exc_value, traceback): - if exc_type is None: - self.result.get() - self.server.stop() - - def test_connection(): - @nsqd_handler + @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' assert socket.recv(1) == '' @@ -75,7 +50,7 @@ def handle(socket, address): ]) def test_read(body): - @nsqd_handler + @mock_server def handle(socket, address): socket.send(struct.pack('>l', len(body))) socket.send(body) @@ -90,7 +65,7 @@ def handle(socket, address): def test_identify(): - @nsqd_handler + @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' assert socket.recv(9) == 'IDENTIFY\n' @@ -110,7 +85,7 @@ def handle(socket, address): def test_negotiation(): - @nsqd_handler + @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' assert socket.recv(9) == 'IDENTIFY\n' @@ -147,7 +122,7 @@ def handle(socket, address): ]) def test_command(command, args, resp): - @nsqd_handler + @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' assert socket.recv(len(resp)) == resp @@ -158,9 +133,40 @@ def handle(socket, address): getattr(conn, command)(*args) +@pytest.mark.parametrize('error,error_class,fatal', [ + ('E_INVALID', errors.NSQInvalid, True), + ('E_BAD_BODY', errors.NSQBadBody, True), + ('E_BAD_TOPIC', errors.NSQBadTopic, True), + ('E_BAD_CHANNEL', errors.NSQBadChannel, True), + ('E_BAD_MESSAGE', errors.NSQBadMessage, True), + ('E_PUT_FAILED', errors.NSQPutFailed, True), + ('E_PUB_FAILED', errors.NSQPubFailed, True), + ('E_MPUB_FAILED', errors.NSQMPubFailed, True), + ('E_FIN_FAILED', errors.NSQFinishFailed, False), + ('E_REQ_FAILED', errors.NSQRequeueFailed, False), + ('E_TOUCH_FAILED', errors.NSQTouchFailed, False), + ('unknown error', errors.NSQException, True), +]) +def test_error(error, error_class, fatal): + + @mock_server + def handle(socket, address): + assert socket.recv(4) == ' V2' + socket.send(mock_response(nsq.FRAME_TYPE_ERROR, error)) + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + conn.connect() + + frame, resp = conn.read_response() + assert frame == nsq.FRAME_TYPE_ERROR + assert isinstance(resp, error_class) + assert conn.is_connected != fatal + + def test_sync_receive_messages(): - @nsqd_handler + @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' assert socket.recv(9) == 'IDENTIFY\n' From 036ad726f727d0039f381f6395507647034e828b Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 29 Jun 2014 21:10:55 -0700 Subject: [PATCH 031/113] Test nsqd hashing. --- tests/test_nsqd.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index b38411d..f5d9eb5 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -164,6 +164,15 @@ def handle(socket, address): assert conn.is_connected != fatal +def test_hashing(): + conn1 = Nsqd('localhost', 1337) + conn2 = Nsqd('localhost', 1337) + assert conn1 == conn2 + + test = {conn1: True} + assert conn2 in test + + def test_sync_receive_messages(): @mock_server From e2ced47f9a847bccbccf6e40a169aaedd07e5aa3 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 29 Jun 2014 23:43:26 -0700 Subject: [PATCH 032/113] Allow keyword arguments to be passed along to connections. --- gnsq/reader.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 1655c31..85e7ca2 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -35,6 +35,7 @@ def __init__( lookupd_poll_jitter=0.3, low_ready_idle_timeout=10, max_backoff_duration=128, + **kwargs ): if not nsqd_tcp_addresses and not lookupd_http_addresses: raise ValueError('must specify at least on nsqd or lookupd') @@ -68,6 +69,7 @@ def __init__( self.lookupd_poll_jitter = lookupd_poll_jitter self.low_ready_idle_timeout = low_ready_idle_timeout self.max_backoff_duration = max_backoff_duration + self.conn_kwargs = kwargs if name: self.name = name @@ -159,7 +161,7 @@ def query_nsqd(self): self.logger.debug('querying nsqd...') for address in self.nsqd_tcp_addresses: address, port = address.split(':') - conn = Nsqd(address, int(port)) + conn = Nsqd(address, int(port), **self.conn_kwargs) self.connect_to_nsqd(conn) def query_lookupd(self): @@ -179,7 +181,8 @@ def query_lookupd(self): conn = Nsqd( producer.get('broadcast_address') or producer['address'], producer['tcp_port'], - producer['http_port'] + producer['http_port'], + **self.conn_kwargs ) self.connect_to_nsqd(conn) From 39fcbd4313068c8cffa8be257a7e23814f221dbd Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 29 Jun 2014 23:51:46 -0700 Subject: [PATCH 033/113] Fixes to contributing guide. --- CONTRIBUTING.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 5d58c78..283e507 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -3,7 +3,7 @@ Contributing ============ Contributions are welcome, and they are greatly appreciated! Every -little bit helps, and credit will always be given. +little bit helps, and credit will always be given. You can contribute in many ways: @@ -36,7 +36,7 @@ is open to whoever wants to implement it. Write Documentation ~~~~~~~~~~~~~~~~~~~ -gnsq could always use more documentation, whether as part of the +gnsq could always use more documentation, whether as part of the official gnsq docs, in docstrings, or even on the web in blog posts, articles, and such. @@ -66,21 +66,21 @@ Ready to contribute? Here's how to set up `gnsq` for local development. $ mkvirtualenv gnsq $ cd gnsq/ - $ python setup.py develop + $ pip install -r requirements.txt 4. Create a branch for local development:: $ git checkout -b name-of-your-bugfix-or-feature - + Now you can make your changes locally. 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: $ flake8 gnsq tests - $ python setup.py test + $ py.test $ tox - To get flake8 and tox, just pip install them into your virtualenv. + To get flake8 and tox, just pip install them into your virtualenv. 6. Commit your changes and push your branch to GitHub:: @@ -108,4 +108,4 @@ Tips To run a subset of tests:: - $ python -m unittest tests.test_gnsq \ No newline at end of file + $ py.test tests/test_basic.py From 7395514751608ce7117adb129a2b525c2a4ab65f Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 30 Jun 2014 00:05:30 -0700 Subject: [PATCH 034/113] Combine installation and usage into the read me. --- README.rst | 21 ++++++++++++++++++++- docs/index.rst | 4 +--- docs/{readme.rst => quickstart.rst} | 0 docs/usage.rst | 7 ------- 4 files changed, 21 insertions(+), 11 deletions(-) rename docs/{readme.rst => quickstart.rst} (100%) delete mode 100644 docs/usage.rst diff --git a/README.rst b/README.rst index a17c675..f5b4a1d 100644 --- a/README.rst +++ b/README.rst @@ -20,4 +20,23 @@ A gevent based NSQ driver for Python. Features -------- -* TODO \ No newline at end of file +* TODO + +Installation +------------ + +At the command line:: + + $ easy_install gnsq + +Or, if you have virtualenvwrapper installed:: + + $ mkvirtualenv gnsq + $ pip install gnsq + +Usage +----- + +To use gnsq in a project:: + + import gnsq diff --git a/docs/index.rst b/docs/index.rst index d104dae..dc5ff2f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,9 +11,7 @@ Contents: .. toctree:: :maxdepth: 2 - readme - installation - usage + quickstart contributing authors history diff --git a/docs/readme.rst b/docs/quickstart.rst similarity index 100% rename from docs/readme.rst rename to docs/quickstart.rst diff --git a/docs/usage.rst b/docs/usage.rst deleted file mode 100644 index 33755bf..0000000 --- a/docs/usage.rst +++ /dev/null @@ -1,7 +0,0 @@ -======== -Usage -======== - -To use gnsq in a project:: - - import gnsq \ No newline at end of file From 268bf62c856cbb89d2793d714f3edef3ea240053 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 30 Jun 2014 00:12:58 -0700 Subject: [PATCH 035/113] Improve cleaning scripts. --- Makefile | 23 +++++++++++++---------- docs/Makefile | 4 ++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index e19ad98..01c55a2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean-pyc clean-build docs clean +.PHONY: clean-pyc clean-build clean-docs docs clean help: @echo "clean-build - remove build artifacts" @@ -11,7 +11,7 @@ help: @echo "release - package and upload a release" @echo "dist - package" -clean: clean-build clean-pyc +clean: clean-build clean-pyc clean-docs rm -fr htmlcov/ clean-build: @@ -20,9 +20,15 @@ clean-build: rm -fr *.egg-info clean-pyc: - find . -name '*.pyc' -exec rm -f {} + - find . -name '*.pyo' -exec rm -f {} + - find . -name '*~' -exec rm -f {} + + find . -type f -name "*.py[co]" -delete + find . -type f -name '*~' -delete + find . -type d -name "__pycache__" -delete + +clean-docs: + rm -f docs/gnsq.rst + rm -f docs/gnsq.stream.rst + rm -f docs/modules.rst + $(MAKE) -C docs clean lint: flake8 gnsq tests @@ -39,11 +45,8 @@ coverage: coverage html open htmlcov/index.html -docs: - rm -f docs/gnsq.rst - rm -f docs/modules.rst +docs: clean-docs sphinx-apidoc -o docs/ gnsq - $(MAKE) -C docs clean $(MAKE) -C docs html open docs/_build/html/index.html @@ -54,4 +57,4 @@ release: clean dist: clean python setup.py sdist python setup.py bdist_wheel - ls -l dist \ No newline at end of file + ls -l dist diff --git a/docs/Makefile b/docs/Makefile index 1005e26..caad866 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -47,7 +47,7 @@ help: @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: - rm -rf $(BUILDDIR)/* + rm -rf $(BUILDDIR) html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @@ -174,4 +174,4 @@ xml: pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." \ No newline at end of file + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." From 0097122edff3175442e5b841247cfc01ddd21b45 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 30 Jun 2014 00:16:38 -0700 Subject: [PATCH 036/113] Fix header size in history. --- HISTORY.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5f35edc..d1ee0b9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,6 @@ History ------- 0.1.0 (2014-01-11) ---------------------- +~~~~~~~~~~~~~~~~~~ -* First release on PyPI. \ No newline at end of file +* First release on PyPI. From 0ab7cd45c4cd8c26bc14ed96ab98c2592e12c8a7 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Tue, 1 Jul 2014 17:28:19 -0700 Subject: [PATCH 037/113] Integration tests for tls, deflate and snappy. --- tests/cert.pem | 26 ++++++++++++++ tests/integration_server.py | 65 +++++++++++++++++++++++++++++++++ tests/key.pem | 27 ++++++++++++++ tests/test_nsqd.py | 72 +++++++++++++++++++++++++++++++++---- 4 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 tests/cert.pem create mode 100644 tests/integration_server.py create mode 100644 tests/key.pem diff --git a/tests/cert.pem b/tests/cert.pem new file mode 100644 index 0000000..ed73acf --- /dev/null +++ b/tests/cert.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEbjCCA1agAwIBAgIJAK6x7y6AwBmLMA0GCSqGSIb3DQEBBQUAMIGAMQswCQYD +VQQGEwJVUzERMA8GA1UECBMITmV3IFlvcmsxFjAUBgNVBAcTDU5ldyBZb3JrIENp +dHkxDDAKBgNVBAoTA05TUTETMBEGA1UEAxMKdGVzdC5sb2NhbDEjMCEGCSqGSIb3 +DQEJARYUbXJlaWZlcnNvbkBnbWFpbC5jb20wHhcNMTMwNjI4MDA0MzQ4WhcNMTYw +NDE3MDA0MzQ4WjCBgDELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMRYw +FAYDVQQHEw1OZXcgWW9yayBDaXR5MQwwCgYDVQQKEwNOU1ExEzARBgNVBAMTCnRl +c3QubG9jYWwxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnX0KB+svwy+yHU2qggz/EaGg +craKShagKo+9M9y5HLM852ngk5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS +25tVP1UIEnT5pBt2TeetLkl199Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQY +shuEJHKeUNDbQKz5X+GjEdkTPO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECz +ydZBgTPThy3uDtHIuCpxCwXd/vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF +7m3/0KbtUcXfy0QHueeuMr11E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QID +AQABo4HoMIHlMB0GA1UdDgQWBBR3HMBws4lmYYSIgwoZsfW+bbgaMjCBtQYDVR0j +BIGtMIGqgBR3HMBws4lmYYSIgwoZsfW+bbgaMqGBhqSBgzCBgDELMAkGA1UEBhMC +VVMxETAPBgNVBAgTCE5ldyBZb3JrMRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MQww +CgYDVQQKEwNOU1ExEzARBgNVBAMTCnRlc3QubG9jYWwxIzAhBgkqhkiG9w0BCQEW +FG1yZWlmZXJzb25AZ21haWwuY29tggkArrHvLoDAGYswDAYDVR0TBAUwAwEB/zAN +BgkqhkiG9w0BAQUFAAOCAQEANOYTbanW2iyV1v4oYpcM/y3TWcQKzSME8D2SGFZb +dbMYU81hH3TTlQdvyeh3FAcdjhKE8Xi/RfNNjEslTBscdKXePGpZg6eXRNJzPP5K +KZPf5u6tcpAeUOKrMqbGwbE+h2QixxG1EoVQtE421szsU2P7nHRTdHzKFRnOerfl +Phm3NocR0P40Rv7WKdxpOvqc+XKf0onTruoVYoPWGpwcLixCG0zu4ZQ23/L/Dy18 +4u70Hbq6O/6kq9FBFaDNp3IhiEdu2Cq6ZplU6bL9XDF27KIEErHwtuqBHVlMG+zB +oH/k9vZvwH7OwAjHdKp+1yeZFLYC8K5hjFIHqcdwpZCNIg== +-----END CERTIFICATE----- diff --git a/tests/integration_server.py b/tests/integration_server.py new file mode 100644 index 0000000..608691c --- /dev/null +++ b/tests/integration_server.py @@ -0,0 +1,65 @@ +import random +import time +import shutil +import subprocess +import tempfile +import requests +import os.path + + +class IntegrationNsqdServer(object): + tls_cert = os.path.join(os.path.dirname(__file__), 'cert.pem') + tls_key = os.path.join(os.path.dirname(__file__), 'key.pem') + + def __init__(self, port=None): + if port is None: + port = random.randint(10000, 65535) + self.port = port + self.data_path = tempfile.mkdtemp() + + @property + def tcp_address(self): + return '127.0.0.1:{}'.format(self.port) + + @property + def http_address(self): + return '127.0.0.1:{}'.format(self.port + 1) + + def is_running(self): + try: + resp = requests.get('http://{}/ping'.format(self.http_address)) + return resp.text == 'OK' + except requests.ConnectionError: + return False + + def wait(self): + while True: + if self.is_running(): + break + time.sleep(0.1) + + def __enter__(self): + print 'running:', ' '.join([ + 'nsqd', + '--tcp-address={}'.format(self.tcp_address), + '--http-address={}'.format(self.http_address), + '--data-path={}'.format(self.data_path), + '--tls-cert={}'.format(self.tls_cert), + '--tls-key={}'.format(self.tls_key), + ]) + + self.nsqd = subprocess.Popen([ + 'nsqd', + '--tcp-address={}'.format(self.tcp_address), + '--http-address={}'.format(self.http_address), + '--data-path={}'.format(self.data_path), + '--tls-cert={}'.format(self.tls_cert), + '--tls-key={}'.format(self.tls_key), + ]) + self.wait() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.nsqd.terminate() + self.nsqd.wait() + shutil.rmtree(self.data_path) diff --git a/tests/key.pem b/tests/key.pem new file mode 100644 index 0000000..9b4db2e --- /dev/null +++ b/tests/key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAnX0KB+svwy+yHU2qggz/EaGgcraKShagKo+9M9y5HLM852ng +k5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS25tVP1UIEnT5pBt2TeetLkl1 +99Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQYshuEJHKeUNDbQKz5X+GjEdkT +PO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECzydZBgTPThy3uDtHIuCpxCwXd +/vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF7m3/0KbtUcXfy0QHueeuMr11 +E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QIDAQABAoIBACvtfKbIywG+hAf4 +ad7skRjx5DcbA2e29+XnQfb9UgTXWd2SgrmoLi5OypBkCTzkKN3mfTo70yZfV8dC +Sxwz+9tfnTz0DssjhKThS+CiaFVCkeOfSfBfKSlCQUVHrSrh18CDhP+yvDlJwQTZ +zSQMfPcsh9bmJe2kqtQP7ZgUp1o+vaB8Sju8YYrO6FllxbdLRGm4pfvvrHIRRmXa +oVHn0ei0JpwoTY9kHYht4LNeJnbP/MCWdmcuv3Gnel7jAlhaKab5aNIGr0Xe7aIQ +iX6mpZ0/Rnt8o/XcTOg8l3ruIdVuySX6SYn08JMnfFkXdNYRVhoV1tC5ElWkaZLf +hPmj2yECgYEAyts0R0b8cZ6HTAyuLm3ilw0s0v0/MM9ZtaqMRilr2WEtAhF0GpHG +TzmGnii0WcTNXD7NTsNcECR/0ZpXPRleMczsL2Juwd4FkQ37h7hdKPseJNrfyHRg +VolOFBX9H14C3wMB9cwdsG4Egw7fE27WCoreEquHgwFxl1zBrXKH088CgYEAxr8w +BKZs0bF7LRrFT5pH8hpMLYHMYk8ZIOfgmEGVBKDQCOERPR9a9kqUss7wl/98LVNK +RnFlyWD6Z0/QcQsLL4LjBeZJ25qEMc6JXm9VGAzhXA1ZkUofVoYCnG+f6KUn8CuJ +/AcV2ZDFsEP10IiQG0hKsceXiwFEvEr8306tMrcCgYBLgnscSR0xAeyk71dq6vZc +ecgEpcX+2kAvclOSzlpZ6WVCjtKkDT0/Qk+M0eQIQkybGLl9pxS+4Yc+s2/jy2yX +pwsHvGE0AvwZeZX2eDcdSRR4bYy9ZixyKdwJeAHnyivRbaIuJ5Opl9pQGpoI9snv +1K9DTdw8dK4exKVHdgl/WwKBgDkmLsuXg4EEtPOyV/xc08VVNIR9Z2T5c7NXmeiO +KyiKiWeUOF3ID2L07S9BfENozq9F3PzGjMtMXJSqibiHwW6nB1rh7mj8VHjx9+Q0 +xVZGFeNfX1r84mgB3uxW2LeQDhzsmB/lda37CC14TU3qhu2hawEV8IijE73FHlOk +Dv+fAoGAI4/XO5o5tNn5Djo8gHmGMCbinUE9+VySxl7wd7PK8w2VSofO88ofixDk +NX94yBYhg5WZcLdPm45RyUnq+WVQYz9IKUrdxLFTH+wxyzUqZCW7jgXCvWV+071q +vqm9C+kndq+18/1VKuCSGWnF7Ay4lbsgPXY2s4VKRxcb3QpZSPU= +-----END RSA PRIVATE KEY----- diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index f5d9eb5..b2be977 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -6,8 +6,10 @@ from gnsq import Nsqd, Message, states, errors from gnsq import protocal as nsq +from gnsq.stream.stream import SSLSocket, DefalteSocket, SnappySocket from mock_server import mock_server +from integration_server import IntegrationNsqdServer def mock_response(frame_type, data): @@ -26,7 +28,6 @@ def mock_response_message(timestamp, attempts, id, body): def test_connection(): - @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' @@ -49,7 +50,6 @@ def handle(socket, address): '{"some": "json data"}', ]) def test_read(body): - @mock_server def handle(socket, address): socket.send(struct.pack('>l', len(body))) @@ -64,7 +64,6 @@ def handle(socket, address): def test_identify(): - @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' @@ -84,7 +83,6 @@ def handle(socket, address): def test_negotiation(): - @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' @@ -121,7 +119,6 @@ def handle(socket, address): ('nop', (), 'NOP\n'), ]) def test_command(command, args, resp): - @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' @@ -148,7 +145,6 @@ def handle(socket, address): ('unknown error', errors.NSQException, True), ]) def test_error(error, error_class, fatal): - @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' @@ -174,7 +170,6 @@ def test_hashing(): def test_sync_receive_messages(): - @mock_server def handle(socket, address): assert socket.recv(4) == ' V2' @@ -219,3 +214,66 @@ def handle(socket, address): assert msg.id == '%016d' % i assert msg.attempts == i assert json.loads(msg.body)['data']['test_key'] == i + + +@pytest.mark.slow +def test_tls(): + with IntegrationNsqdServer() as server: + print { + 'keyfile': server.tls_key, + 'certfile': server.tls_cert, + } + conn = Nsqd( + address='127.0.0.1', + tcp_port=server.port, + tls_v1=True, + tls_options={ + 'keyfile': server.tls_key, + 'certfile': server.tls_cert, + } + ) + conn.connect() + assert conn.state == states.CONNECTED + + resp = conn.identify() + assert isinstance(resp, dict) + assert resp['tls_v1'] is True + assert isinstance(conn.stream.socket, SSLSocket) + + frame, data = conn.read_response() + assert frame == nsq.FRAME_TYPE_RESPONSE + assert data == 'OK' + + +@pytest.mark.slow +def test_deflate(): + with IntegrationNsqdServer() as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.port, deflate=True) + conn.connect() + assert conn.state == states.CONNECTED + + resp = conn.identify() + assert isinstance(resp, dict) + assert resp['deflate'] is True + assert isinstance(conn.stream.socket, DefalteSocket) + + frame, data = conn.read_response() + assert frame == nsq.FRAME_TYPE_RESPONSE + assert data == 'OK' + + +@pytest.mark.slow +def test_snappy(): + with IntegrationNsqdServer() as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.port, snappy=True) + conn.connect() + assert conn.state == states.CONNECTED + + resp = conn.identify() + assert isinstance(resp, dict) + assert resp['snappy'] is True + assert isinstance(conn.stream.socket, SnappySocket) + + frame, data = conn.read_response() + assert frame == nsq.FRAME_TYPE_RESPONSE + assert data == 'OK' From e1239cd4a1ba58b8abcbcd279450d725de76f38e Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 10:29:26 -0700 Subject: [PATCH 038/113] Add reader integration tests. --- Makefile | 3 ++ tests/conftest.py | 14 ++++++++++ tests/integration_server.py | 48 +++++++++++++++++--------------- tests/test_nsqd.py | 20 ++++++++------ tests/test_reader.py | 55 +++++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 30 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_reader.py diff --git a/Makefile b/Makefile index 01c55a2..e104457 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,9 @@ lint: test: py.test +test-slow: + py.test --runslow + test-all: tox diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..736b2ec --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + '--runslow', + action='store_true', + help='run slow tests' + ) + + +def pytest_runtest_setup(item): + if 'slow' in item.keywords and not item.config.getoption('--runslow'): + pytest.skip('need --runslow option to run') diff --git a/tests/integration_server.py b/tests/integration_server.py index 608691c..10b8eca 100644 --- a/tests/integration_server.py +++ b/tests/integration_server.py @@ -11,19 +11,29 @@ class IntegrationNsqdServer(object): tls_cert = os.path.join(os.path.dirname(__file__), 'cert.pem') tls_key = os.path.join(os.path.dirname(__file__), 'key.pem') - def __init__(self, port=None): - if port is None: - port = random.randint(10000, 65535) - self.port = port + def __init__(self, address=None, tcp_port=None, http_port=None): + if address is None: + address = '127.0.0.1' + + if tcp_port is None: + tcp_port = random.randint(10000, 65535) + tcp_port = 1234 + + if http_port is None: + http_port = tcp_port + 1 + + self.address = address + self.tcp_port = tcp_port + self.http_port = http_port self.data_path = tempfile.mkdtemp() @property def tcp_address(self): - return '127.0.0.1:{}'.format(self.port) + return '{}:{}'.format(self.address, self.tcp_port) @property def http_address(self): - return '127.0.0.1:{}'.format(self.port + 1) + return '{}:{}'.format(self.address, self.http_port) def is_running(self): try: @@ -38,24 +48,18 @@ def wait(self): break time.sleep(0.1) - def __enter__(self): - print 'running:', ' '.join([ + def cmd(self): + return [ 'nsqd', - '--tcp-address={}'.format(self.tcp_address), - '--http-address={}'.format(self.http_address), - '--data-path={}'.format(self.data_path), - '--tls-cert={}'.format(self.tls_cert), - '--tls-key={}'.format(self.tls_key), - ]) + '--tcp-address', self.tcp_address, + '--http-address', self.http_address, + '--data-path', self.data_path, + '--tls-cert', self.tls_cert, + '--tls-key', self.tls_key, + ] - self.nsqd = subprocess.Popen([ - 'nsqd', - '--tcp-address={}'.format(self.tcp_address), - '--http-address={}'.format(self.http_address), - '--data-path={}'.format(self.data_path), - '--tls-cert={}'.format(self.tls_cert), - '--tls-key={}'.format(self.tls_key), - ]) + def __enter__(self): + self.nsqd = subprocess.Popen(self.cmd()) self.wait() return self diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index b2be977..0a01d15 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -219,13 +219,9 @@ def handle(socket, address): @pytest.mark.slow def test_tls(): with IntegrationNsqdServer() as server: - print { - 'keyfile': server.tls_key, - 'certfile': server.tls_cert, - } conn = Nsqd( - address='127.0.0.1', - tcp_port=server.port, + address=server.address, + tcp_port=server.tcp_port, tls_v1=True, tls_options={ 'keyfile': server.tls_key, @@ -248,7 +244,11 @@ def test_tls(): @pytest.mark.slow def test_deflate(): with IntegrationNsqdServer() as server: - conn = Nsqd(address='127.0.0.1', tcp_port=server.port, deflate=True) + conn = Nsqd( + address=server.address, + tcp_port=server.tcp_port, + deflate=True + ) conn.connect() assert conn.state == states.CONNECTED @@ -265,7 +265,11 @@ def test_deflate(): @pytest.mark.slow def test_snappy(): with IntegrationNsqdServer() as server: - conn = Nsqd(address='127.0.0.1', tcp_port=server.port, snappy=True) + conn = Nsqd( + address=server.address, + tcp_port=server.tcp_port, + snappy=True + ) conn.connect() assert conn.state == states.CONNECTED diff --git a/tests/test_reader.py b/tests/test_reader.py new file mode 100644 index 0000000..8128666 --- /dev/null +++ b/tests/test_reader.py @@ -0,0 +1,55 @@ +import pytest +from gnsq import Nsqd, Reader +from gnsq.errors import NSQSocketError +from integration_server import IntegrationNsqdServer + + +@pytest.mark.slow +def test_messages(): + with IntegrationNsqdServer() as server: + + class Accounting(object): + count = 0 + total = 500 + error = None + + conn = Nsqd( + address=server.address, + tcp_port=server.tcp_port, + http_port=server.http_port, + ) + + for _ in xrange(Accounting.total): + conn.publish_http('test', 'danger zone!') + + reader = Reader( + topic='test', + channel='test', + nsqd_tcp_addresses=[server.tcp_address], + max_in_flight=100, + ) + + @reader.on_exception.connect + def error_handler(reader, conn, message, error): + if isinstance(error, NSQSocketError): + return + Accounting.error = error + reader.close() + + @reader.on_message.connect + def handler(reader, conn, message): + assert message.body == 'danger zone!' + + Accounting.count += 1 + if Accounting.count == Accounting.total: + reader.close() + + try: + reader.start() + except NSQSocketError: + pass + + if Accounting.error: + raise Accounting.error + + assert Accounting.count == Accounting.total From d3eda401344ec6f5d5c4873da8f150f657bd118a Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 10:29:53 -0700 Subject: [PATCH 039/113] Fix syntax. --- gnsq/reader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 85e7ca2..852cc4f 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -99,9 +99,11 @@ def __init__( self.conn_workers = {} def start(self, block=True): - if not self.state == INIT: + if self.state != INIT: + self.logger.debug('{} all ready started'.format(self.name)) return + self.logger.debug('starting {}...'.format(self.name)) self.state = RUNNING self.query_nsqd() From ab55425d9faacbf190b3d68eb2e4f860654f4208 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 10:30:10 -0700 Subject: [PATCH 040/113] Don't block on close. This causes deadlock. --- gnsq/reader.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 852cc4f..1eabd40 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -116,21 +116,18 @@ def start(self, block=True): if block: self.join() - def close(self, block=True): + def close(self): if not self.is_running: return self.state = CLOSED for worker in self.workers: - worker.kill(block=block) + worker.kill() for conn in self.conns: conn.close_stream() - if block: - self.join() - def join(self, timeout=None, raise_error=False): gevent.joinall(self.workers, timeout, raise_error) gevent.joinall(self.conn_workers.values(), timeout, raise_error) @@ -440,7 +437,12 @@ def handle_message(self, conn, message): except Exception as error: msg = '[{}] caught exception while handling message'.format(conn) self.logger.exception(msg) - self.on_exception(self, conn=conn, message=message, error=error) + self.on_exception.send( + self, + conn=conn, + message=message, + error=error, + ) message.requeue(self.requeue_delay) From 7fe9cd67a7a95a00260c318e77ffd4abaeea8b2f Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 10:39:17 -0700 Subject: [PATCH 041/113] Reader signals should not send connection. The connection is attached to the message if it is needed. --- gnsq/reader.py | 22 ++++++---------------- tests/test_reader.py | 4 ++-- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 1eabd40..5e586df 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -409,11 +409,11 @@ def handle_connection_failure(self, conn): def handle_response(self, conn, response): self.logger.debug('[{}] response: {}'.format(conn, response)) - self.on_response.send(self, conn=conn, response=response) + self.on_response.send(self, response=response) def handle_error(self, conn, error): self.logger.debug('[{}] error: {}'.format(conn, error)) - self.on_error.send(self, conn=conn, error=error) + self.on_error.send(self, error=error) def handle_message(self, conn, message): self.logger.debug('[{}] got message: {}'.format(conn, message.id)) @@ -426,7 +426,7 @@ def handle_message(self, conn, message): return message.finish() try: - self.on_message.send(self, conn=conn, message=message) + self.on_message.send(self, message=message) if not self.async: message.finish() return @@ -437,12 +437,7 @@ def handle_message(self, conn, message): except Exception as error: msg = '[{}] caught exception while handling message'.format(conn) self.logger.exception(msg) - self.on_exception.send( - self, - conn=conn, - message=message, - error=error, - ) + self.on_exception.send(self, message=message, error=error) message.requeue(self.requeue_delay) @@ -450,19 +445,14 @@ def handle_finish(self, conn, message_id): self.logger.debug('[{}] finished message: {}'.format(conn, message_id)) self.backoff.success() self.handle_backoff() - self.on_finish.send(self, conn=conn, message_id=message_id) + self.on_finish.send(self, message_id=message_id) def handle_requeue(self, conn, message_id, timeout): msg = '[{}] requeued message: {} ({})' self.logger.debug(msg.format(conn, message_id, timeout)) self.backoff.failure() self.handle_backoff() - self.on_requeue.send( - self, - conn=conn, - message_id=message_id, - timeout=timeout - ) + self.on_requeue.send(self, message_id=message_id, timeout=timeout) def handle_backoff(self): if self.state == BACKOFF: diff --git a/tests/test_reader.py b/tests/test_reader.py index 8128666..8dfbeea 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -30,14 +30,14 @@ class Accounting(object): ) @reader.on_exception.connect - def error_handler(reader, conn, message, error): + def error_handler(reader, message, error): if isinstance(error, NSQSocketError): return Accounting.error = error reader.close() @reader.on_message.connect - def handler(reader, conn, message): + def handler(reader, message): assert message.body == 'danger zone!' Accounting.count += 1 From bc933f49b446feee9d216e10bb06d9bba873363a Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 10:43:57 -0700 Subject: [PATCH 042/113] Add basic usage docs. --- README.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index f5b4a1d..c1e7220 100644 --- a/README.rst +++ b/README.rst @@ -17,11 +17,6 @@ A gevent based NSQ driver for Python. * Free software: BSD license * Documentation: http://gnsq.readthedocs.org. -Features --------- - -* TODO - Installation ------------ @@ -29,7 +24,7 @@ At the command line:: $ easy_install gnsq -Or, if you have virtualenvwrapper installed:: +Or even better, if you have virtualenvwrapper installed:: $ mkvirtualenv gnsq $ pip install gnsq @@ -40,3 +35,10 @@ Usage To use gnsq in a project:: import gnsq + reader = gnsq.Reader('topic', 'channel', 'localhost:4150') + + @reader.on_message.connect + def handler(reader, message): + do_work(message.body) + + reader.start() From 68f86907debfedcc088a751e8a9221dfc5eafecf Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 10:57:42 -0700 Subject: [PATCH 043/113] Don't close if not connected. --- gnsq/nsqd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index fdabf45..2823801 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -96,6 +96,9 @@ def connect(self): self.send(nsq.MAGIC_V2) def close_stream(self): + if not self.is_connected: + return + self.stream.close() self.state = DISCONNECTED self.on_close.send(self) From 403101d1ac374acbea13c6da802ecdc8b4abcfeb Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 11:03:34 -0700 Subject: [PATCH 044/113] Iterate directly over conns. --- gnsq/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 5e586df..601515a 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -211,7 +211,7 @@ def complete_backoff(self): self.logger.info('backoff complete, resuming normal operation') count = self.connection_max_in_flight - for conn in self.conns.values(): + for conn in self.conns: self.send_ready(conn, count) def _poll_lookupd(self): From 7139374601013443ab8d845463e4d1461407223d Mon Sep 17 00:00:00 2001 From: Ian Preston Date: Wed, 2 Jul 2014 15:59:08 -0700 Subject: [PATCH 045/113] s/total_ready/total_ready_count --- gnsq/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 601515a..1064df4 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -265,9 +265,9 @@ def redistribute_ready_state(self): conn.ready(0) if self.state == THROTTLED: - max_in_flight = 1 - self.total_ready + max_in_flight = 1 - self.total_ready_count else: - max_in_flight = self.max_in_flight - self.total_ready + max_in_flight = self.max_in_flight - self.total_ready_count if max_in_flight <= 0: return From 1bc99a80e7b1aed84b0c16f9cdcb3b4bacccee64 Mon Sep 17 00:00:00 2001 From: Ian Preston Date: Wed, 2 Jul 2014 16:49:49 -0700 Subject: [PATCH 046/113] s/conn/conns --- gnsq/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 1064df4..c731c1e 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -281,7 +281,7 @@ def redistribute_ready_state(self): # have had RDY 1 because it would be overly complicated and not actually # worth it (ie. given enough redistribution rounds it doesn't matter). conns = list(self.conns) - conns = random.sample(conns, min(max_in_flight, len(self.conn))) + conns = random.sample(conns, min(max_in_flight, len(self.conns))) for conn in conns: self.logger.info('[{}] redistributing RDY'.format(conn)) From dbd6420cc502a7d74ea9d39eb47e62968c61a311 Mon Sep 17 00:00:00 2001 From: Ian Preston Date: Wed, 2 Jul 2014 17:05:20 -0700 Subject: [PATCH 047/113] In random_ready_conn, pass list to random.choice rather than generator --- gnsq/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index c731c1e..c06b2ed 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -297,7 +297,7 @@ def random_ready_conn(self, conn): return conn self.last_random_ready = time.time() - return random.choice(c for c in self.conns if not c.ready_count) + return random.choice([c for c in self.conns if not c.ready_count]) def update_ready(self, conn): if self.state in (BACKOFF, THROTTLED): From ec197f4ed6d3c4f5a77bbfb2a712ca8b1fbbf3c1 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 17:26:50 -0700 Subject: [PATCH 048/113] Fix version. --- gnsq/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnsq/version.py b/gnsq/version.py index 0267974..b9df18c 100644 --- a/gnsq/version.py +++ b/gnsq/version.py @@ -1,2 +1,2 @@ # also update in setup.py -__version__ = '0.0.1' +__version__ = '0.1.0' From 99540cbb99062f62a01838387e1f9b3c0247ebbc Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 17:28:08 -0700 Subject: [PATCH 049/113] Don't update ready until message have been finished or requeued. --- gnsq/reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index c06b2ed..19b0f16 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -417,7 +417,6 @@ def handle_error(self, conn, error): def handle_message(self, conn, message): self.logger.debug('[{}] got message: {}'.format(conn, message.id)) - self.update_ready(conn) if self.max_tries and message.attempts > self.max_tries: msg = "giving up on message '{}' after max tries {}" @@ -444,6 +443,7 @@ def handle_message(self, conn, message): def handle_finish(self, conn, message_id): self.logger.debug('[{}] finished message: {}'.format(conn, message_id)) self.backoff.success() + self.update_ready(conn) self.handle_backoff() self.on_finish.send(self, message_id=message_id) @@ -451,6 +451,7 @@ def handle_requeue(self, conn, message_id, timeout): msg = '[{}] requeued message: {} ({})' self.logger.debug(msg.format(conn, message_id, timeout)) self.backoff.failure() + self.update_ready(conn) self.handle_backoff() self.on_requeue.send(self, message_id=message_id, timeout=timeout) From 0c253964bcd715e0bc20b2e78c165907f0eb0a4c Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 18:09:15 -0700 Subject: [PATCH 050/113] Send rdy 1 to only the first connection if not in backoff or throttle. --- gnsq/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 19b0f16..65a9544 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -353,8 +353,8 @@ def connect_to_nsqd(self, conn): conn.subscribe(self.topic, self.channel) - # Send RDY 1 if we're not backing off or we're the first connection - if self.state != BACKOFF or not self.conns: + # Send RDY 1 if we're not backing off and we're the first connection + if self.state not in (BACKOFF, THROTTLED) and not self.conns: self.send_ready(conn, 1) except NSQException as error: From 3f0969180b0953a3f3c1c5368440ef80a27a6189 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Wed, 2 Jul 2014 18:10:00 -0700 Subject: [PATCH 051/113] Use redistribute ready when finishing back off. --- gnsq/reader.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 65a9544..997ef45 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -209,10 +209,7 @@ def start_backoff(self): def complete_backoff(self): self.state = RUNNING self.logger.info('backoff complete, resuming normal operation') - - count = self.connection_max_in_flight - for conn in self.conns: - self.send_ready(conn, count) + self.redistribute_ready_state() def _poll_lookupd(self): delay = self.lookupd_poll_interval * self.lookupd_poll_jitter From afb1bb1804c22cb45407f638b9e2d0211e94c22b Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Thu, 3 Jul 2014 09:51:41 -0700 Subject: [PATCH 052/113] Revert "Use redistribute ready when finishing back off." This reverts commit 3f0969180b0953a3f3c1c5368440ef80a27a6189. --- gnsq/reader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 997ef45..65a9544 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -209,7 +209,10 @@ def start_backoff(self): def complete_backoff(self): self.state = RUNNING self.logger.info('backoff complete, resuming normal operation') - self.redistribute_ready_state() + + count = self.connection_max_in_flight + for conn in self.conns: + self.send_ready(conn, count) def _poll_lookupd(self): delay = self.lookupd_poll_interval * self.lookupd_poll_jitter From 093f0e3a31782e6a87509fcf7933aa9f5c79a7c0 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Thu, 3 Jul 2014 09:55:17 -0700 Subject: [PATCH 053/113] Revert "Send rdy 1 to only the first connection if not in backoff or throttle." This reverts commit 0c253964bcd715e0bc20b2e78c165907f0eb0a4c. --- gnsq/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 65a9544..19b0f16 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -353,8 +353,8 @@ def connect_to_nsqd(self, conn): conn.subscribe(self.topic, self.channel) - # Send RDY 1 if we're not backing off and we're the first connection - if self.state not in (BACKOFF, THROTTLED) and not self.conns: + # Send RDY 1 if we're not backing off or we're the first connection + if self.state != BACKOFF or not self.conns: self.send_ready(conn, 1) except NSQException as error: From 30dfcec8525175584555f2af10cfab240b396f96 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Thu, 3 Jul 2014 10:15:29 -0700 Subject: [PATCH 054/113] Properly distribute readys. --- gnsq/reader.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 19b0f16..4016b00 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -151,9 +151,26 @@ def connection_max_in_flight(self): def total_ready_count(self): return sum(c.ready_count for c in self.conns) + @property + def total_in_flight(self): + return sum(c.in_flight for c in self.conns) + + @property + def total_in_flight_or_ready(self): + return self.total_in_flight + self.total_ready_count + def send_ready(self, conn, count): - if (self.total_ready_count + count) > self.max_in_flight: + if self.state == BACKOFF: + return + + if self.state == THROTTLED and self.total_in_flight_or_ready: + return + + if (self.total_in_flight_or_ready + count) > self.max_in_flight: + if not (conn.ready_count or conn.in_flight): + gevent.spawn_later(5, self.send_ready, conn, count) return + conn.ready(count) def query_nsqd(self): @@ -265,9 +282,9 @@ def redistribute_ready_state(self): conn.ready(0) if self.state == THROTTLED: - max_in_flight = 1 - self.total_ready_count + max_in_flight = 1 - self.total_in_flight_or_ready else: - max_in_flight = self.max_in_flight - self.total_ready_count + max_in_flight = self.max_in_flight - self.total_in_flight_or_ready if max_in_flight <= 0: return @@ -352,10 +369,7 @@ def connect_to_nsqd(self, conn): ]).format(conn, conn.max_ready_count, self.max_in_flight)) conn.subscribe(self.topic, self.channel) - - # Send RDY 1 if we're not backing off or we're the first connection - if self.state != BACKOFF or not self.conns: - self.send_ready(conn, 1) + self.send_ready(conn, 1) except NSQException as error: msg = '[{}] connection failed ({!r})'.format(conn, error) From f74d9bbdf2339a40bcfcef8cea11adabc3d64c75 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Thu, 3 Jul 2014 14:07:13 -0700 Subject: [PATCH 055/113] Not socket closing is not a failure if we are closing. --- gnsq/reader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gnsq/reader.py b/gnsq/reader.py index 4016b00..8f47792 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -392,6 +392,8 @@ def _listen(self, conn): try: conn.listen() except NSQException as error: + if self.state == CLOSED: + return msg = '[{}] connection lost ({!r})'.format(conn, error) self.logger.warning(msg) From 02ccea98f8ade43acad5243cef9f00804d52f1d3 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Fri, 4 Jul 2014 13:40:48 -0700 Subject: [PATCH 056/113] Validate we're running before write operations. --- gnsq/reader.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 8f47792..7b2f673 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -160,7 +160,7 @@ def total_in_flight_or_ready(self): return self.total_in_flight + self.total_ready_count def send_ready(self, conn, count): - if self.state == BACKOFF: + if self.state in (BACKOFF, CLOSED): return if self.state == THROTTLED and self.total_in_flight_or_ready: @@ -442,7 +442,7 @@ def handle_message(self, conn, message): try: self.on_message.send(self, message=message) - if not self.async: + if self.is_running and not self.async: message.finish() return @@ -454,7 +454,8 @@ def handle_message(self, conn, message): self.logger.exception(msg) self.on_exception.send(self, message=message, error=error) - message.requeue(self.requeue_delay) + if self.is_running: + message.requeue(self.requeue_delay) def handle_finish(self, conn, message_id): self.logger.debug('[{}] finished message: {}'.format(conn, message_id)) @@ -472,7 +473,7 @@ def handle_requeue(self, conn, message_id, timeout): self.on_requeue.send(self, message_id=message_id, timeout=timeout) def handle_backoff(self): - if self.state == BACKOFF: + if self.state in (BACKOFF, CLOSED): return if self.state == THROTTLED and self.backoff.is_reset(): From d83f6dd05804e0fecbcbd7212275e94fdc1d689c Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Fri, 4 Jul 2014 13:41:19 -0700 Subject: [PATCH 057/113] Warn when starting an already started reader. --- gnsq/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 7b2f673..b46a975 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -100,7 +100,7 @@ def __init__( def start(self, block=True): if self.state != INIT: - self.logger.debug('{} all ready started'.format(self.name)) + self.logger.warn('{} all ready started'.format(self.name)) return self.logger.debug('starting {}...'.format(self.name)) From d69427499f7ae235a79d021714ef3e3c4eb6904c Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Fri, 4 Jul 2014 13:41:35 -0700 Subject: [PATCH 058/113] Fix giving up signal signature. --- gnsq/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index b46a975..0f386e1 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -437,7 +437,7 @@ def handle_message(self, conn, message): if self.max_tries and message.attempts > self.max_tries: msg = "giving up on message '{}' after max tries {}" self.logger.warning(msg.format(message.id, self.max_tries)) - self.on_giving_up.send(self, conn, message) + self.on_giving_up.send(self, message=message) return message.finish() try: From 1cbc291911ffacd0eb055e35e432ec4863d846df Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Fri, 4 Jul 2014 13:54:21 -0700 Subject: [PATCH 059/113] Added reader pool class. --- gnsq/__init__.py | 2 ++ gnsq/pool.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 gnsq/pool.py diff --git a/gnsq/__init__.py b/gnsq/__init__.py index b1bb16d..421c5d0 100644 --- a/gnsq/__init__.py +++ b/gnsq/__init__.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from .reader import Reader +from .pool import ReaderPool from .nsqd import Nsqd from .lookupd import Lookupd from .message import Message @@ -12,6 +13,7 @@ __all__ = [ 'Reader', + 'ReaderPool', 'Nsqd', 'Lookupd', 'Message', diff --git a/gnsq/pool.py b/gnsq/pool.py new file mode 100644 index 0000000..7f82e29 --- /dev/null +++ b/gnsq/pool.py @@ -0,0 +1,31 @@ +import multiprocessing +import gevent +from gevent.queue import Queue +from .reader import Reader + + +class ReaderPool(Reader): + def __init__(self, topic, channel, pool_size=None, **kwargs): + super(ReaderPool, self).__init__(topic, channel, **kwargs) + + if pool_size is None: + pool_size = 2 * multiprocessing.cpu_count() + 1 + + self.queue = Queue() + self.pool_size = pool_size + + def _run(self): + for conn, message in self.queue: + super(ReaderPool, self).handle_message(conn, message) + + def handle_message(self, conn, message): + self.queue.put((conn, message)) + + def start(self, block=True): + super(ReaderPool, self).start(block=False) + + workers = [gevent.spawn(self._run) for _ in xrange(self.pool_size)] + self.workers.extend(workers) + + if block: + self.join() From 319869cfd5375064ca70eb86e3daf3481b4d9cfe Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Fri, 4 Jul 2014 14:19:33 -0700 Subject: [PATCH 060/113] Allow passing in a message handler to the reader constructor. --- gnsq/reader.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gnsq/reader.py b/gnsq/reader.py index 0f386e1..d38ec12 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -27,6 +27,7 @@ def __init__( nsqd_tcp_addresses=[], lookupd_http_addresses=[], name=None, + message_handler=None, async=False, max_tries=5, max_in_flight=1, @@ -92,6 +93,9 @@ def __init__( self.on_giving_up = blinker.Signal() self.on_exception = blinker.Signal() + if message_handler is not None: + self.on_message.connect(message_handler) + self.conns = set() self.pending = set() From fdd11526dfa30ee47e03390fd555d7d419b583da Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Fri, 4 Jul 2014 14:31:13 -0700 Subject: [PATCH 061/113] Don't wait forever for nsqd to start. Its not going to happen man. --- tests/integration_server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration_server.py b/tests/integration_server.py index 10b8eca..ec169c5 100644 --- a/tests/integration_server.py +++ b/tests/integration_server.py @@ -43,10 +43,11 @@ def is_running(self): return False def wait(self): - while True: + for attempt in xrange(10): if self.is_running(): - break - time.sleep(0.1) + return + time.sleep(0.01 * pow(2, attempt)) + raise RuntimeError('unable to start nsqd') def cmd(self): return [ From 01da68495b8f90865354371fe374e94b8dba93cc Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Fri, 4 Jul 2014 15:26:00 -0700 Subject: [PATCH 062/113] Combine pool with reader via max_concurrency argument. --- gnsq/__init__.py | 2 -- gnsq/pool.py | 31 ---------------------- gnsq/reader.py | 68 ++++++++++++++++++++++++++++++++++++------------ 3 files changed, 52 insertions(+), 49 deletions(-) delete mode 100644 gnsq/pool.py diff --git a/gnsq/__init__.py b/gnsq/__init__.py index 421c5d0..b1bb16d 100644 --- a/gnsq/__init__.py +++ b/gnsq/__init__.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from .reader import Reader -from .pool import ReaderPool from .nsqd import Nsqd from .lookupd import Lookupd from .message import Message @@ -13,7 +12,6 @@ __all__ = [ 'Reader', - 'ReaderPool', 'Nsqd', 'Lookupd', 'Message', diff --git a/gnsq/pool.py b/gnsq/pool.py deleted file mode 100644 index 7f82e29..0000000 --- a/gnsq/pool.py +++ /dev/null @@ -1,31 +0,0 @@ -import multiprocessing -import gevent -from gevent.queue import Queue -from .reader import Reader - - -class ReaderPool(Reader): - def __init__(self, topic, channel, pool_size=None, **kwargs): - super(ReaderPool, self).__init__(topic, channel, **kwargs) - - if pool_size is None: - pool_size = 2 * multiprocessing.cpu_count() + 1 - - self.queue = Queue() - self.pool_size = pool_size - - def _run(self): - for conn, message in self.queue: - super(ReaderPool, self).handle_message(conn, message) - - def handle_message(self, conn, message): - self.queue.put((conn, message)) - - def start(self, block=True): - super(ReaderPool, self).start(block=False) - - workers = [gevent.spawn(self._run) for _ in xrange(self.pool_size)] - self.workers.extend(workers) - - if block: - self.join() diff --git a/gnsq/reader.py b/gnsq/reader.py index d38ec12..3577059 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -6,6 +6,8 @@ from itertools import cycle from collections import defaultdict +from multiprocessing import cpu_count +from gevent.queue import Queue from .lookupd import Lookupd from .nsqd import Nsqd @@ -31,6 +33,7 @@ def __init__( async=False, max_tries=5, max_in_flight=1, + max_concurrency=0, requeue_delay=0, lookupd_poll_interval=60, lookupd_poll_jitter=0.3, @@ -41,22 +44,11 @@ def __init__( if not nsqd_tcp_addresses and not lookupd_http_addresses: raise ValueError('must specify at least on nsqd or lookupd') - if isinstance(nsqd_tcp_addresses, basestring): - self.nsqd_tcp_addresses = [nsqd_tcp_addresses] - elif isinstance(nsqd_tcp_addresses, (list, tuple)): - self.nsqd_tcp_addresses = set(nsqd_tcp_addresses) - else: - raise TypeError('nsqd_tcp_addresses must be a list, set or tuple') - - if isinstance(lookupd_http_addresses, basestring): - lookupd_http_addresses = [lookupd_http_addresses] - elif isinstance(lookupd_http_addresses, (list, tuple)): - lookupd_http_addresses = list(lookupd_http_addresses) - random.shuffle(lookupd_http_addresses) - else: - msg = 'lookupd_http_addresses must be a list, set or tuple' - raise TypeError(msg) + nsqd_tcp_addresses = self._get_nsqds(nsqd_tcp_addresses) + lookupd_http_addresses = self._get_lookupds(lookupd_http_addresses) + random.shuffle(lookupd_http_addresses) + self.nsqd_tcp_addresses = nsqd_tcp_addresses self.lookupds = [Lookupd(a) for a in lookupd_http_addresses] self.iterlookupds = cycle(self.lookupds) @@ -65,6 +57,7 @@ def __init__( self.async = async self.max_tries = max_tries self.max_in_flight = max_in_flight + self.max_concurrency = max_concurrency self.requeue_delay = requeue_delay self.lookupd_poll_interval = lookupd_poll_interval self.lookupd_poll_jitter = lookupd_poll_jitter @@ -96,12 +89,40 @@ def __init__( if message_handler is not None: self.on_message.connect(message_handler) + if max_concurrency < 0: + max_concurrency = cpu_count() + + if max_concurrency: + self.queue = Queue() + else: + self.queue = None + self.conns = set() self.pending = set() self.workers = [] self.conn_workers = {} + def _get_nsqds(self, nsqd_tcp_addresses): + if isinstance(nsqd_tcp_addresses, basestring): + return set([nsqd_tcp_addresses]) + + elif isinstance(nsqd_tcp_addresses, (list, tuple, set)): + return set(nsqd_tcp_addresses) + + raise TypeError('nsqd_tcp_addresses must be a list, set or tuple') + + def _get_lookupds(self, lookupd_http_addresses): + if isinstance(lookupd_http_addresses, basestring): + return [lookupd_http_addresses] + + elif isinstance(lookupd_http_addresses, (list, tuple)): + lookupd_http_addresses = list(lookupd_http_addresses) + return lookupd_http_addresses + + msg = 'lookupd_http_addresses must be a list, set or tuple' + raise TypeError(msg) + def start(self, block=True): if self.state != INIT: self.logger.warn('{} all ready started'.format(self.name)) @@ -117,6 +138,9 @@ def start(self, block=True): self.workers.append(gevent.spawn(self._poll_ready)) + for _ in xrange(self.max_concurrency): + self.workers.append(gevent.spawn(self._run)) + if block: self.join() @@ -356,10 +380,14 @@ def connect_to_nsqd(self, conn): conn.on_response.connect(self.handle_response) conn.on_error.connect(self.handle_error) - conn.on_message.connect(self.handle_message) conn.on_finish.connect(self.handle_finish) conn.on_requeue.connect(self.handle_requeue) + if self.max_concurrency: + conn.on_message.connect(self.queue_message) + else: + conn.on_message.connect(self.handle_message) + self.pending.add(conn) try: @@ -403,6 +431,14 @@ def _listen(self, conn): self.handle_connection_failure(conn) + def _run(self): + for conn, message in self.queue: + self.handle_message(conn, message) + + def queue_message(self, conn, message): + self.logger.debug('[{}] queueing message: {}'.format(conn, message.id)) + self.queue.put((conn, message)) + def handle_connection_success(self, conn): self.conns.add(conn) self.conn_workers[conn] = gevent.spawn(self._listen, conn) From 95867858210a5a0bae33e54adb15ea532365cdab Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 11:53:16 -0700 Subject: [PATCH 063/113] Fix python2.6 compatibility. --- gnsq/__init__.py | 4 ++- gnsq/backofftimer.py | 1 + gnsq/errors.py | 1 + gnsq/httpclient.py | 1 + gnsq/lookupd.py | 1 + gnsq/message.py | 1 + gnsq/nsqd.py | 12 ++++--- gnsq/protocal.py | 2 ++ gnsq/reader.py | 72 +++++++++++++++++++------------------ gnsq/stream/stream.py | 1 + setup.py | 3 +- tests/integration_server.py | 6 ++-- tox.ini | 6 ++-- 13 files changed, 64 insertions(+), 47 deletions(-) diff --git a/gnsq/__init__.py b/gnsq/__init__.py index b1bb16d..37fd507 100644 --- a/gnsq/__init__.py +++ b/gnsq/__init__.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import from .reader import Reader from .nsqd import Nsqd from .lookupd import Lookupd from .message import Message from .backofftimer import BackoffTimer +from .version import __version__ __author__ = 'Trevor Olson' __email__ = 'trevor@heytrevor.com' -__version__ = '0.1.0' +__version__ = __version__ __all__ = [ 'Reader', diff --git a/gnsq/backofftimer.py b/gnsq/backofftimer.py index 68f35ec..2099624 100644 --- a/gnsq/backofftimer.py +++ b/gnsq/backofftimer.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import random diff --git a/gnsq/errors.py b/gnsq/errors.py index 45edffb..b219e88 100644 --- a/gnsq/errors.py +++ b/gnsq/errors.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import socket diff --git a/gnsq/httpclient.py b/gnsq/httpclient.py index 1e98bde..3edf15e 100644 --- a/gnsq/httpclient.py +++ b/gnsq/httpclient.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import requests from .errors import NSQException diff --git a/gnsq/lookupd.py b/gnsq/lookupd.py index c19b6b6..afe0623 100644 --- a/gnsq/lookupd.py +++ b/gnsq/lookupd.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from .httpclient import HTTPClient from . import protocal as nsq diff --git a/gnsq/message.py b/gnsq/message.py index e95a1c1..33dff9a 100644 --- a/gnsq/message.py +++ b/gnsq/message.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from .errors import NSQException diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 2823801..235dc31 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import blinker import time from gevent import socket @@ -18,7 +20,7 @@ HOSTNAME = socket.gethostname() SHORTNAME = HOSTNAME.split('.')[0] -USERAGENT = 'gnsq/{}'.format(__version__) +USERAGENT = 'gnsq/%s' % __version__ class Nsqd(HTTPClient): @@ -124,7 +126,7 @@ def read_response(self): self.last_response = time.time() if frame not in self._frame_handlers: - raise errors.NSQFrameError('unknown frame {}'.format(frame)) + raise errors.NSQFrameError('unknown frame %d' % frame) frame_handler = self._frame_handlers[frame] processed_data = frame_handler(data) @@ -215,8 +217,8 @@ def identify(self): except ValueError: self.close_stream() - msg = 'failed to parse IDENTIFY response JSON from nsqd: {!r}' - raise errors.NSQException(msg.format(data)) + msg = 'failed to parse IDENTIFY response JSON from nsqd: %r' + raise errors.NSQException(msg % data) self.max_ready_count = data.get('max_rdy_count', self.max_ready_count) @@ -267,7 +269,7 @@ def nop(self): @property def base_url(self): - return 'http://{}:{}/'.format(self.address, self.http_port) + return 'http://%s:%s/' % (self.address, self.http_port) def _check_connection(self): if self.http_port: diff --git a/gnsq/protocal.py b/gnsq/protocal.py index afc8f33..98e9c3a 100644 --- a/gnsq/protocal.py +++ b/gnsq/protocal.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import re import struct diff --git a/gnsq/reader.py b/gnsq/reader.py index 3577059..b761035 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import logging import random import gevent @@ -68,7 +70,7 @@ def __init__( if name: self.name = name else: - self.name = '{}.{}.{}'.format(__name__, self.topic, self.channel) + self.name = '%s.%s.%s' % (__name__, self.topic, self.channel) self._need_ready_redistributed = False self.last_random_ready = time.time() @@ -125,10 +127,10 @@ def _get_lookupds(self, lookupd_http_addresses): def start(self, block=True): if self.state != INIT: - self.logger.warn('{} all ready started'.format(self.name)) + self.logger.warn('%s all ready started' % self.name) return - self.logger.debug('starting {}...'.format(self.name)) + self.logger.debug('starting %s...' % self.name) self.state = RUNNING self.query_nsqd() @@ -214,11 +216,11 @@ def query_lookupd(self): try: producers = lookupd.lookup(self.topic)['producers'] - self.logger.debug('found {} producers'.format(len(producers))) + self.logger.debug('found %d producers' % len(producers)) except Exception as error: - msg = 'Failed to lookup {} on {} ({})' - self.logger.warn(msg.format(self.topic, lookupd.address, error)) + msg = 'Failed to lookup %s on %s (%s)' + self.logger.warn(msg % (self.topic, lookupd.address, error)) return for producer in producers: @@ -240,7 +242,7 @@ def start_backoff(self): conn.ready(0) interval = self.backoff.get_interval() - self.logger.info('backing off for {} seconds'.format(interval)) + self.logger.info('backing off for %s seconds' % interval) gevent.sleep(interval) self.state = THROTTLED @@ -248,7 +250,7 @@ def start_backoff(self): return conn = self.random_connection() - self.logger.info('[{}] testing backoff state with RDY 1'.format(conn)) + self.logger.info('[%s] testing backoff state with RDY 1' % conn) self.send_ready(conn, 1) def complete_backoff(self): @@ -304,9 +306,7 @@ def redistribute_ready_state(self): if (time.time() - conn.last_message) < self.low_ready_idle_timeout: continue - msg = '[{}] idle connection, giving up RDY count'.format(conn) - self.logger.info(msg) - + self.logger.info('[%s] idle connection, giving up RDY count' % conn) conn.ready(0) if self.state == THROTTLED: @@ -329,7 +329,7 @@ def redistribute_ready_state(self): conns = random.sample(conns, min(max_in_flight, len(self.conns))) for conn in conns: - self.logger.info('[{}] redistributing RDY'.format(conn)) + self.logger.info('[%s] redistributing RDY' % conn) self.send_ready(conn, 1) def random_ready_conn(self, conn): @@ -369,14 +369,14 @@ def connect_to_nsqd(self, conn): return if conn in self.conns: - self.logger.debug('[{}] already connected'.format(conn)) + self.logger.debug('[%s] already connected' % conn) return if conn in self.pending: - self.logger.debug('[{}] already pending'.format(conn)) + self.logger.debug('[%s] already pending' % conn) return - self.logger.debug('[{}] connecting...'.format(conn)) + self.logger.debug('[%s] connecting...' % conn) conn.on_response.connect(self.handle_response) conn.on_error.connect(self.handle_error) @@ -395,17 +395,22 @@ def connect_to_nsqd(self, conn): conn.identify() if conn.max_ready_count < self.max_in_flight: - self.logger.warning(' '.join([ - '[{}] max RDY count {} < reader max in flight {},', + msg = ' '.join([ + '[%s] max RDY count %d < reader max in flight %d,', 'truncation possible' - ]).format(conn, conn.max_ready_count, self.max_in_flight)) + ]) + + self.logger.warning(msg % ( + conn, + conn.max_ready_count, + self.max_in_flight + )) conn.subscribe(self.topic, self.channel) self.send_ready(conn, 1) except NSQException as error: - msg = '[{}] connection failed ({!r})'.format(conn, error) - self.logger.debug(msg) + self.logger.debug('[%s] connection failed (%r)' % (conn, error)) self.handle_connection_failure(conn) return @@ -417,7 +422,7 @@ def connect_to_nsqd(self, conn): conn.close_stream() return - self.logger.info('[{}] connection successful'.format(conn)) + self.logger.info('[%s] connection successful' % conn) self.handle_connection_success(conn) def _listen(self, conn): @@ -426,8 +431,7 @@ def _listen(self, conn): except NSQException as error: if self.state == CLOSED: return - msg = '[{}] connection lost ({!r})'.format(conn, error) - self.logger.warning(msg) + self.logger.warning('[%s] connection lost (%r)' % (conn, error)) self.handle_connection_failure(conn) @@ -436,7 +440,7 @@ def _run(self): self.handle_message(conn, message) def queue_message(self, conn, message): - self.logger.debug('[{}] queueing message: {}'.format(conn, message.id)) + self.logger.debug('[%s] queueing message: %s' % (conn, message.id)) self.queue.put((conn, message)) def handle_connection_success(self, conn): @@ -460,23 +464,23 @@ def handle_connection_failure(self, conn): return seconds = self.conn_backoffs[conn].failure().get_interval() - self.logger.debug('[{}] retrying in {}s'.format(conn, seconds)) + self.logger.debug('[%s] retrying in %ss' % (conn, seconds)) gevent.spawn_later(seconds, self.connect_to_nsqd, conn) def handle_response(self, conn, response): - self.logger.debug('[{}] response: {}'.format(conn, response)) + self.logger.debug('[%s] response: %s' % (conn, response)) self.on_response.send(self, response=response) def handle_error(self, conn, error): - self.logger.debug('[{}] error: {}'.format(conn, error)) + self.logger.debug('[%s] error: %s' % (conn, error)) self.on_error.send(self, error=error) def handle_message(self, conn, message): - self.logger.debug('[{}] got message: {}'.format(conn, message.id)) + self.logger.debug('[%s] got message: %s' % (conn, message.id)) if self.max_tries and message.attempts > self.max_tries: - msg = "giving up on message '{}' after max tries {}" - self.logger.warning(msg.format(message.id, self.max_tries)) + msg = "giving up on message '%s' after max tries %d" + self.logger.warning(msg % (message.id, self.max_tries)) self.on_giving_up.send(self, message=message) return message.finish() @@ -490,7 +494,7 @@ def handle_message(self, conn, message): pass except Exception as error: - msg = '[{}] caught exception while handling message'.format(conn) + msg = '[%s] caught exception while handling message' % conn self.logger.exception(msg) self.on_exception.send(self, message=message, error=error) @@ -498,15 +502,15 @@ def handle_message(self, conn, message): message.requeue(self.requeue_delay) def handle_finish(self, conn, message_id): - self.logger.debug('[{}] finished message: {}'.format(conn, message_id)) + self.logger.debug('[%s] finished message: %s' % (conn, message_id)) self.backoff.success() self.update_ready(conn) self.handle_backoff() self.on_finish.send(self, message_id=message_id) def handle_requeue(self, conn, message_id, timeout): - msg = '[{}] requeued message: {} ({})' - self.logger.debug(msg.format(conn, message_id, timeout)) + msg = '[%s] requeued message: %s (%s)' + self.logger.debug(msg % (conn, message_id, timeout)) self.backoff.failure() self.update_ready(conn) self.handle_backoff() diff --git a/gnsq/stream/stream.py b/gnsq/stream/stream.py index a90770d..1c26094 100644 --- a/gnsq/stream/stream.py +++ b/gnsq/stream/stream.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + from resource import getpagesize from errno import ENOTCONN diff --git a/setup.py b/setup.py index a4914fb..f636020 100755 --- a/setup.py +++ b/setup.py @@ -36,10 +36,11 @@ zip_safe=False, keywords='gnsq', classifiers=[ - 'Development Status :: 2 - Pre-Alpha', + 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Natural Language :: English', + 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', ] ) diff --git a/tests/integration_server.py b/tests/integration_server.py index ec169c5..924d509 100644 --- a/tests/integration_server.py +++ b/tests/integration_server.py @@ -29,15 +29,15 @@ def __init__(self, address=None, tcp_port=None, http_port=None): @property def tcp_address(self): - return '{}:{}'.format(self.address, self.tcp_port) + return '%s:%d' % (self.address, self.tcp_port) @property def http_address(self): - return '{}:{}'.format(self.address, self.http_port) + return '%s:%d' % (self.address, self.http_port) def is_running(self): try: - resp = requests.get('http://{}/ping'.format(self.http_address)) + resp = requests.get('http://%s/ping' % self.http_address) return resp.text == 'OK' except requests.ConnectionError: return False diff --git a/tox.ini b/tox.ini index 66f014c..1fbe6de 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,14 @@ [tox] -envlist = py26, py27, py33, py34 +envlist = py26, py27 [testenv] setenv = PYTHONPATH = {toxinidir}:{toxinidir}/gnsq -commands = python setup.py test +commands = py.test --runslow deps = -r{toxinidir}/requirements.txt [flake8] max-line-length = 80 exclude = tests/* -max-complexity = 10 \ No newline at end of file +max-complexity = 10 From f7566e30b84503e529b50ea54a788ff52fe5c825 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 11:56:40 -0700 Subject: [PATCH 064/113] Cleanup setup.py --- setup.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index f636020..e10f584 100755 --- a/setup.py +++ b/setup.py @@ -10,12 +10,6 @@ readme = open('README.rst').read() history = open('HISTORY.rst').read().replace('.. :changelog:', '') -requirements = [ - 'gevent', - 'blinker', - 'requests', -] - setup( name='gnsq', @@ -25,13 +19,14 @@ author='Trevor Olson', author_email='trevor@heytrevor.com', url='https://github.com/wtolson/gnsq', - packages=[ - 'gnsq', - ], - package_dir={'gnsq': - 'gnsq'}, + packages=['gnsq'], + package_dir={'gnsq': 'gnsq'}, include_package_data=True, - install_requires=requirements, + install_requires=[ + 'gevent', + 'blinker', + 'requests', + ], license="BSD", zip_safe=False, keywords='gnsq', From 6abf9a232dea12f5ae7818cc0e930814132261ad Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 12:07:55 -0700 Subject: [PATCH 065/113] Add travis config. --- .travis.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..75d6015 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,16 @@ +# Config file for automatic testing at travis-ci.org + +language: python + +python: + - "2.7" + - "2.6" + +install: + - sudo apt-get install libsnappy-dev libevent-dev + - pip install -r requirements.txt + - wget https://s3.amazonaws.com/bitly-downloads/nsq/nsq-0.2.28.linux-amd64.go1.2.1.tar.gz + - tar zxvf nsq-0.2.28.linux-amd64.go1.2.1.tar.gz + - sudo cp nsq-0.2.28.linux-amd64.go1.2.1/bin/nsqd nsq-0.2.28.linux-amd64.go1.2.1/bin/nsqlookupd /usr/local/bin + +script: py.test --runslow From b14f6e6a95a457852fef608b9dcd2cc606b1f2f0 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 12:26:46 -0700 Subject: [PATCH 066/113] Include basic usage in index page. --- README.rst | 6 +++++- docs/index.rst | 7 +++++-- docs/installation.rst | 12 ------------ docs/quickstart.rst | 1 - 4 files changed, 10 insertions(+), 16 deletions(-) delete mode 100644 docs/installation.rst delete mode 100644 docs/quickstart.rst diff --git a/README.rst b/README.rst index c1e7220..3d5cac8 100644 --- a/README.rst +++ b/README.rst @@ -12,7 +12,7 @@ gnsq :target: https://pypi.python.org/pypi/gnsq -A gevent based NSQ driver for Python. +A `gevent`_ based `NSQ`_ driver for Python. * Free software: BSD license * Documentation: http://gnsq.readthedocs.org. @@ -42,3 +42,7 @@ To use gnsq in a project:: do_work(message.body) reader.start() + + +.. _gevent: http://gevent.org/ +.. _NSQ: http://nsq.io/ diff --git a/docs/index.rst b/docs/index.rst index dc5ff2f..fed14ef 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,9 +4,12 @@ contain the root `toctree` directive. Welcome to gnsq's documentation! -====================================== +================================ -Contents: +.. include:: ../README.rst + +Contents +======== .. toctree:: :maxdepth: 2 diff --git a/docs/installation.rst b/docs/installation.rst deleted file mode 100644 index 64d8c79..0000000 --- a/docs/installation.rst +++ /dev/null @@ -1,12 +0,0 @@ -============ -Installation -============ - -At the command line:: - - $ easy_install gnsq - -Or, if you have virtualenvwrapper installed:: - - $ mkvirtualenv gnsq - $ pip install gnsq \ No newline at end of file diff --git a/docs/quickstart.rst b/docs/quickstart.rst deleted file mode 100644 index 6b2b3ec..0000000 --- a/docs/quickstart.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../README.rst \ No newline at end of file From 02c31a8752f14f9868334d1c8178212531dd4882 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 12:47:46 -0700 Subject: [PATCH 067/113] utf-8! --- gnsq/backofftimer.py | 1 + gnsq/errors.py | 1 + gnsq/httpclient.py | 1 + gnsq/lookupd.py | 1 + gnsq/message.py | 1 + gnsq/nsqd.py | 1 + gnsq/protocal.py | 1 + gnsq/reader.py | 1 + gnsq/states.py | 1 + gnsq/stream/__init__.py | 1 + gnsq/stream/compression.py | 2 +- gnsq/stream/defalte.py | 2 +- gnsq/stream/snappy.py | 2 +- gnsq/stream/stream.py | 1 + gnsq/version.py | 1 + 15 files changed, 15 insertions(+), 3 deletions(-) diff --git a/gnsq/backofftimer.py b/gnsq/backofftimer.py index 2099624..7e8a60a 100644 --- a/gnsq/backofftimer.py +++ b/gnsq/backofftimer.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import import random diff --git a/gnsq/errors.py b/gnsq/errors.py index b219e88..33c6e24 100644 --- a/gnsq/errors.py +++ b/gnsq/errors.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import import socket diff --git a/gnsq/httpclient.py b/gnsq/httpclient.py index 3edf15e..7eae101 100644 --- a/gnsq/httpclient.py +++ b/gnsq/httpclient.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import import requests from .errors import NSQException diff --git a/gnsq/lookupd.py b/gnsq/lookupd.py index afe0623..9e73a40 100644 --- a/gnsq/lookupd.py +++ b/gnsq/lookupd.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import from .httpclient import HTTPClient from . import protocal as nsq diff --git a/gnsq/message.py b/gnsq/message.py index 33dff9a..97eedd9 100644 --- a/gnsq/message.py +++ b/gnsq/message.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import from .errors import NSQException diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 235dc31..23fe5d7 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import import blinker diff --git a/gnsq/protocal.py b/gnsq/protocal.py index 98e9c3a..2c41dc5 100644 --- a/gnsq/protocal.py +++ b/gnsq/protocal.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import import re diff --git a/gnsq/reader.py b/gnsq/reader.py index b761035..5865368 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import import logging diff --git a/gnsq/states.py b/gnsq/states.py index 93ac79a..24a42a0 100644 --- a/gnsq/states.py +++ b/gnsq/states.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """Connection states.""" INIT = 0 diff --git a/gnsq/stream/__init__.py b/gnsq/stream/__init__.py index 0ed74f5..8842acf 100644 --- a/gnsq/stream/__init__.py +++ b/gnsq/stream/__init__.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import from .stream import Stream diff --git a/gnsq/stream/compression.py b/gnsq/stream/compression.py index 95528a0..f756786 100644 --- a/gnsq/stream/compression.py +++ b/gnsq/stream/compression.py @@ -1,5 +1,5 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import - from errno import EWOULDBLOCK from gnsq.errors import NSQSocketError diff --git a/gnsq/stream/defalte.py b/gnsq/stream/defalte.py index f445ae9..8940e2c 100644 --- a/gnsq/stream/defalte.py +++ b/gnsq/stream/defalte.py @@ -1,5 +1,5 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import - import zlib from .compression import CompressionSocket diff --git a/gnsq/stream/snappy.py b/gnsq/stream/snappy.py index aeeca3e..a3042c8 100644 --- a/gnsq/stream/snappy.py +++ b/gnsq/stream/snappy.py @@ -1,5 +1,5 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import - import snappy from .compression import CompressionSocket diff --git a/gnsq/stream/stream.py b/gnsq/stream/stream.py index 1c26094..5fe8f7c 100644 --- a/gnsq/stream/stream.py +++ b/gnsq/stream/stream.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import from resource import getpagesize diff --git a/gnsq/version.py b/gnsq/version.py index b9df18c..6865456 100644 --- a/gnsq/version.py +++ b/gnsq/version.py @@ -1,2 +1,3 @@ +# -*- coding: utf-8 -*- # also update in setup.py __version__ = '0.1.0' From 3c0958227c19c9a52d59f024784508125c6cdba2 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 13:30:29 -0700 Subject: [PATCH 068/113] Add docs outline. --- docs/index.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index fed14ef..69182ea 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,7 +14,9 @@ Contents .. toctree:: :maxdepth: 2 - quickstart + reader + nsqd + lookupd contributing authors history From c5cfaa4ed016e1512e56bcf55753329a1aadf961 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 13:30:58 -0700 Subject: [PATCH 069/113] Basic documentation for reader methods, signals and parameters. --- docs/reader.rst | 6 +++ gnsq/reader.py | 102 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 docs/reader.rst diff --git a/docs/reader.rst b/docs/reader.rst new file mode 100644 index 0000000..45964eb --- /dev/null +++ b/docs/reader.rst @@ -0,0 +1,6 @@ +:class:`Reader` -- high-level consumer +-------------------------------------- + +.. autoclass:: gnsq.Reader + :members: + :inherited-members: diff --git a/gnsq/reader.py b/gnsq/reader.py index 5865368..ce7b347 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -25,6 +25,102 @@ class Reader(object): + """High level NSQ consumer. + + + **Signals:** + + .. data:: on_message(reader, message) + :noindex: + + Sent when the reader receives a message. The `message_handler` param is + connected to this signal. + + .. data:: on_finish(reader, message_id) + :noindex: + + Sent when a message is successfully finished. + + .. data:: on_requeue(reader, message_id, timeout) + :noindex: + + Sent when a message is has been requeued. + + .. data:: on_giving_up(reader, message) + :noindex: + + Sent when a message has exceeded the maximum number of attempts + (`max_tries`) and will no longer be requeued. + + .. data:: on_response(reader, response) + :noindex: + + Sent when the reader receives a response frame from a connection. + + .. data:: on_error(reader, error) + :noindex: + + Sent when the reader receives an error frame from a connection. + + .. data:: on_exception(reader, message, error) + :noindex: + + Sent when an exception is caught while handling a message. + + + :param topic: specifies the desired NSQ topic + + :param channel: specifies the desired NSQ channel + + :param nsqd_tcp_addresses: a sequence of string addresses of the nsqd + instances this reader should connect to + + :param lookupd_http_addresses: a sequence of string addresses of the + nsqlookupd instances this reader should query for producers of the + specified topic + + :param name: a string that is used for logging messages (defaults to + 'gnsq.reader.topic.channel') + + :param message_handler: the callable that will be executed for each message + received + + :param async: consider the message handling to be async. The message will + not automatically be finished after the handler returns and must + manually be called + + :param max_tries: the maximum number of attempts the reader will make to + process a message after which messages will be automatically discarded + + :param max_in_flight: the maximum number of messages this reader will + pipeline for processing. this value will be divided evenly amongst the + configured/discovered nsqd producers + + :param max_concurrency: the maximum number of messages that will be handled + concurrently. Defaults to the number of nsqd connections. Setting + `max_concurrency` to `-1` will use the systems cpu count. + + :param requeue_delay: the default delay to use when requeueing a failed + message + + :param lookupd_poll_interval: the amount of time in seconds between querying + all of the supplied nsqlookupd instances. a random amount of time based + on thie value will be initially introduced in order to add jitter when + multiple readers are running + + :param lookupd_poll_jitter: The maximum fractional amount of jitter to add + to the lookupd pool loop. This helps evenly distribute requests even if + multiple consumers restart at the same time. + + :param low_ready_idle_timeout: the amount of time in seconds to wait for a + message from a producer when in a state where RDY counts are + re-distributed (ie. max_in_flight < num_producers) + + :param max_backoff_duration: the maximum time we will allow a backoff state + to last in seconds + + :param \*\*kwargs: passed to :class:`gnsq.Nsqd` initialization + """ def __init__( self, topic, @@ -127,6 +223,7 @@ def _get_lookupds(self, lookupd_http_addresses): raise TypeError(msg) def start(self, block=True): + """Start discovering and listing to connections.""" if self.state != INIT: self.logger.warn('%s all ready started' % self.name) return @@ -148,6 +245,7 @@ def start(self, block=True): self.join() def close(self): + """Immediately close all connections and stop workers.""" if not self.is_running: return @@ -160,15 +258,18 @@ def close(self): conn.close_stream() def join(self, timeout=None, raise_error=False): + """Block until all connections have closed and workers stopped.""" gevent.joinall(self.workers, timeout, raise_error) gevent.joinall(self.conn_workers.values(), timeout, raise_error) @property def is_running(self): + """Check if reader is currently running.""" return self.state in (RUNNING, BACKOFF, THROTTLED) @property def is_starved(self): + """Check if reader is currently starved for messages.""" for conn in self.conns: if conn.in_flight >= max(conn.last_ready * 0.85, 1): return True @@ -359,6 +460,7 @@ def random_connection(self): return random.choice(list(self.conns)) def publish(self, topic, message): + """Publish a message to a random connection.""" conn = self.random_connection() if conn is None: raise NSQNoConnections() From 31340c9879686dfdb99cac2b7c8439fbece3136f Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 15:39:32 -0700 Subject: [PATCH 070/113] Description of reader mechanics. --- docs/reader.rst | 4 ++-- gnsq/reader.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/reader.rst b/docs/reader.rst index 45964eb..2d123ac 100644 --- a/docs/reader.rst +++ b/docs/reader.rst @@ -1,5 +1,5 @@ -:class:`Reader` -- high-level consumer --------------------------------------- +Reader: high-level consumer +--------------------------- .. autoclass:: gnsq.Reader :members: diff --git a/gnsq/reader.py b/gnsq/reader.py index ce7b347..32ab829 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -27,6 +27,17 @@ class Reader(object): """High level NSQ consumer. + A Reader will connect to the nsqd tcp addresses or poll the provided + nsqlookupd http addresses for the configured topic and send signals to + message handlers connected to the `on_message` signal or provided by + `message_handler`. + + Messages will automatically be finished when the message handle returns + unless the readers `async` flag is set to `True`. If an exception occurs or + :class:`gnsq.errors.NSQRequeueMessage` is raised, the message will be requeued. + + The Reader will handle backing off of failed messages up to a configurable + `max_interval` as well as automatically reconnecting to dropped connections. **Signals:** From 57a0dbc0294c4e55e0ddad15ae69936022815b30 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 15:39:49 -0700 Subject: [PATCH 071/113] Add signals documentation. --- docs/index.rst | 1 + docs/signals.rst | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 docs/signals.rst diff --git a/docs/index.rst b/docs/index.rst index 69182ea..3131fdf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ Contents reader nsqd lookupd + signals contributing authors history diff --git a/docs/signals.rst b/docs/signals.rst new file mode 100644 index 0000000..b25dff5 --- /dev/null +++ b/docs/signals.rst @@ -0,0 +1,28 @@ +Signals +------- + +Both :doc:`Reader ` and :doc:`Nsqd ` classes expose various +signals provided by the `Blinker`_ library. + +Subscribing to signals +~~~~~~~~~~~~~~~~~~~~~~ + +To subscribe to a signal, you can use the +:meth:`~blinker.base.Signal.connect` method of a signal. The first +argument is the function that should be called when the signal is emitted, +the optional second argument specifies a sender. To unsubscribe from a +signal, you can use the :meth:`~blinker.base.Signal.disconnect` method. :: + + def error_handler(reader, error): + print 'Got on error:', error + + reader.on_error.connect(error_handler) + +You can also easily subscribe to signals by using the +:meth:`~blinker.base.NamedSignal.connect` decorator:: + + @reader.on_giving_up.connect + def handle_giving_up(reader, message): + print 'Giving up on:', message.id + +.. _Blinker: https://pypi.python.org/pypi/blinker From 3c3e846ccc3b59c432fa924e86f4f4bc9dd4711b Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 16:06:37 -0700 Subject: [PATCH 072/113] Add nsqd docs page. --- docs/nsqd.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/nsqd.rst diff --git a/docs/nsqd.rst b/docs/nsqd.rst new file mode 100644 index 0000000..7286a10 --- /dev/null +++ b/docs/nsqd.rst @@ -0,0 +1,6 @@ +Nsqd: individual nsqd connection +--------------- + +.. autoclass:: gnsq.Nsqd + :members: + :inherited-members: From 091b6a5c76bb48ae45ef5c01cb6df86076534a74 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 16:07:05 -0700 Subject: [PATCH 073/113] Add nsqd method docs. --- gnsq/nsqd.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 23fe5d7..1427c16 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -85,9 +85,11 @@ def __init__( @property def is_connected(self): + """Check if the client is currently connected.""" return self.state == CONNECTED def connect(self): + """Initialize connection to the nsqd.""" if self.state not in (INIT, DISCONNECTED): return @@ -99,6 +101,7 @@ def connect(self): self.send(nsq.MAGIC_V2) def close_stream(self): + """Close the underlying socket.""" if not self.is_connected: return @@ -122,6 +125,11 @@ def _read_response(self): raise def read_response(self): + """Read an individual response from nsqd. + + :returns: tuple of the frame type and the processed data. + """ + response = self._read_response() frame, data = nsq.unpack_response(response) self.last_response = time.time() @@ -162,6 +170,7 @@ def finish_inflight(self): self.in_flight -= 1 def listen(self): + """Listen to incoming responses until the connection closes.""" while self.is_connected: self.read_response() @@ -175,6 +184,11 @@ def upgrade_to_defalte(self): self.stream.upgrade_to_defalte(self.deflate_level) def identify(self): + """Send the clients `IDENTIFY` command to nsqd. + + :returns: nsqd response data if there was feature negotiation, + otherwise `None` + """ self.send(nsq.identify({ # nsqd <0.2.28 'short_id': self.client_id, @@ -236,36 +250,45 @@ def identify(self): return data def subscribe(self, topic, channel): + """Subscribe to a nsq `topic` and `channel`.""" self.send(nsq.subscribe(topic, channel)) def publish_tcp(self, topic, data): + """Publish a message to the given topic over tcp.""" self.send(nsq.publish(topic, data)) def multipublish_tcp(self, topic, messages): + """Publish an iterable of messages to the given topic over tcp.""" self.send(nsq.multipublish(topic, messages)) def ready(self, count): + """Indicate you are ready to receive `count` messages.""" self.last_ready = count self.ready_count = count self.send(nsq.ready(count)) def finish(self, message_id): + """Finish a message (indicate successful processing).""" self.send(nsq.finish(message_id)) self.finish_inflight() self.on_finish.send(self, message_id=message_id) def requeue(self, message_id, timeout=0): + """Re-queue a message (indicate failure to process).""" self.send(nsq.requeue(message_id, timeout)) self.finish_inflight() self.on_requeue.send(self, message_id=message_id, timeout=timeout) def touch(self, message_id): + """Reset the timeout for an in-flight message.""" self.send(nsq.touch(message_id)) def close(self): + """Indicate no more messages should be sent.""" self.send(nsq.close()) def nop(self): + """Send no-op to nsqd. Used to keep connection alive.""" self.send(nsq.nop()) @property @@ -278,6 +301,7 @@ def _check_connection(self): raise errors.NSQException(-1, 'no http port') def publish_http(self, topic, data): + """Publish a message to the given topic over http.""" nsq.assert_valid_topic_name(topic) return self._check_api( self.url('put'), @@ -286,6 +310,7 @@ def publish_http(self, topic, data): ) def multipublish_http(self, topic, messages): + """Publish an iterable of messages to the given topic over http.""" nsq.assert_valid_topic_name(topic) for message in messages: @@ -302,6 +327,7 @@ def multipublish_http(self, topic, messages): ) def create_topic(self, topic): + """Create a topic.""" nsq.assert_valid_topic_name(topic) return self._json_api( self.url('create_topic'), @@ -309,6 +335,7 @@ def create_topic(self, topic): ) def delete_topic(self, topic): + """Delete a topic.""" nsq.assert_valid_topic_name(topic) return self._json_api( self.url('delete_topic'), @@ -316,6 +343,7 @@ def delete_topic(self, topic): ) def create_channel(self, topic, channel): + """Create a channel for an existing topic.""" nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) return self._json_api( @@ -324,6 +352,7 @@ def create_channel(self, topic, channel): ) def delete_channel(self, topic, channel): + """Delete an existing channel for an existing topic.""" nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) return self._json_api( @@ -332,6 +361,7 @@ def delete_channel(self, topic, channel): ) def empty_topic(self, topic): + """Empty all the queued messages for an existing topic.""" nsq.assert_valid_topic_name(topic) return self._json_api( self.url('empty_topic'), @@ -339,6 +369,7 @@ def empty_topic(self, topic): ) def empty_channel(self, topic, channel): + """Empty all the queued messages for an existing channel.""" nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) return self._json_api( @@ -347,6 +378,10 @@ def empty_channel(self, topic, channel): ) def pause_channel(self, topic, channel): + """Pause message flow to all channels on an existing topic. + + Messages will queue at topic. + """ nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) return self._json_api( @@ -355,6 +390,7 @@ def pause_channel(self, topic, channel): ) def unpause_channel(self, topic, channel): + """Resume message flow to channels of an existing, paused, topic.""" nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) return self._json_api( @@ -363,21 +399,37 @@ def unpause_channel(self, topic, channel): ) def stats(self): + """Return internal instrumented statistics.""" return self._json_api(self.url('stats'), params={'format': 'json'}) def ping(self): + """Monitoring endpoint. + + :returns: should return `"OK"`, otherwise raises an exception. + """ return self._check_api(self.url('ping')) def info(self): + """Returns version information.""" return self._json_api(self.url('info')) def publish(self, topic, data): + """Publish a message. + + If connected, the message will be sent over tcp. Otherwise it will + fall back to http. + """ if self.is_connected: return self.publish_tcp(topic, data) else: return self.publish_http(topic, data) def multipublish(self, topic, messages): + """Publish an iterable of messages in one roundtrip. + + If connected, the messages will be sent over tcp. Otherwise it will + fall back to http. + """ if self.is_connected: return self.multipublish_tcp(topic, messages) else: From 07e8b8719d79c30e92367d090915e712176e07c0 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 16:08:08 -0700 Subject: [PATCH 074/113] Display None as defaults for client_id and hostname in docs. --- gnsq/nsqd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 1427c16..4b6ed9a 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -31,8 +31,8 @@ def __init__( tcp_port=4150, http_port=4151, timeout=60.0, - client_id=SHORTNAME, - hostname=HOSTNAME, + client_id=None, + hostname=None, heartbeat_interval=30, output_buffer_size=16 * 1024, output_buffer_timeout=250, @@ -49,8 +49,8 @@ def __init__( self.http_port = http_port self.timeout = timeout - self.client_id = client_id - self.hostname = hostname + self.client_id = client_id or SHORTNAME + self.hostname = hostname or HOSTNAME self.heartbeat_interval = 1000 * heartbeat_interval self.output_buffer_size = output_buffer_size self.output_buffer_timeout = output_buffer_timeout From 284c2c3954be9a9b0d35bf398f721b61c1dee1f8 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 16:09:20 -0700 Subject: [PATCH 075/113] Stub message page in index. --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 3131fdf..28e1ec8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ Contents reader nsqd lookupd + message signals contributing authors From e032654f5fa0c2092d84ef069f8d5ac5dab74f13 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 16:27:18 -0700 Subject: [PATCH 076/113] Add signals and parameter docs to nsqd. --- gnsq/nsqd.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 4b6ed9a..a4ceefa 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -25,6 +25,87 @@ class Nsqd(HTTPClient): + """Low level object representing a TCP or HTTP connection to nsqd. + + **Signals:** + + .. data:: on_message(conn, message) + :noindex: + + Sent when the connections receives a message. + + .. data:: on_finish(conn, message_id) + :noindex: + + Sent when a message is successfully finished. + + .. data:: on_requeue(conn, message_id, timeout) + :noindex: + + Sent when a message is has been requeued. + + .. data:: on_response(conn, response) + :noindex: + + Sent when the connections receives a response. + + .. data:: on_error(conn, error) + :noindex: + + Sent when the connections receives an error frame. + + .. data:: on_close(conn) + :noindex: + + Sent when the connections stream is closed. + + :param address: the host or ip address of the nsqd + + :param tcp_port: the nsqd tcp port to connect to + + :param http_port: the nsqd http port to connect to + + :param timeout: the timeout for read/write operations (in seconds) + + :param client_id: an identifier used to disambiguate this client (defaults + to the first part of the hostname) + + :param hostname: the hostname where the client is deployed (defaults to the + clients hostname) + + :param heartbeat_interval: the amount of time in seconds to negotiate with + the connected producers to send heartbeats (requires nsqd 0.2.19+) + + :param output_buffer_size: size of the buffer (in bytes) used by nsqd for + buffering writes to this connection + + :param output_buffer_timeout: timeout (in ms) used by nsqd before flushing + buffered writes (set to 0 to disable). Warning: configuring clients with + an extremely low (< 25ms) output_buffer_timeout has a significant effect + on nsqd CPU usage (particularly with > 50 clients connected). + + :param tls_v1: enable TLS v1 encryption (requires nsqd 0.2.22+) + + :param tls_options: dictionary of options to pass to `ssl.wrap_socket() + `_ + + :param snappy: enable Snappy stream compression (requires nsqd 0.2.23+) + + :param deflate: enable deflate stream compression (requires nsqd 0.2.23+) + + :param deflate_level: configure the deflate compression level for this + connection (requires nsqd 0.2.23+) + + :param sample_rate: take only a sample of the messages being sent to the + client. Not setting this or setting it to 0 will ensure you get all the + messages destined for the client. Sample rate can be greater than 0 or + less than 100 and the client will receive that percentage of the message + traffic. (requires nsqd 0.2.25+) + + :param user_agent: a string identifying the agent for this client in the + spirit of HTTP (default: ``/``) (requires + nsqd 0.2.25+) + """ def __init__( self, address='127.0.0.1', @@ -184,7 +265,7 @@ def upgrade_to_defalte(self): self.stream.upgrade_to_defalte(self.deflate_level) def identify(self): - """Send the clients `IDENTIFY` command to nsqd. + """Update client metadata on the server and negotiate features. :returns: nsqd response data if there was feature negotiation, otherwise `None` From 81c268b657c2461f940cad81256ae0029f815bd6 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 16:37:00 -0700 Subject: [PATCH 077/113] Add docs for message. --- docs/message.rst | 6 ++++++ gnsq/message.py | 12 ++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 docs/message.rst diff --git a/docs/message.rst b/docs/message.rst new file mode 100644 index 0000000..d08d312 --- /dev/null +++ b/docs/message.rst @@ -0,0 +1,6 @@ +NSQ Message +----------- + +.. autoclass:: gnsq.Message + :members: + :inherited-members: diff --git a/gnsq/message.py b/gnsq/message.py index 97eedd9..1ac72ea 100644 --- a/gnsq/message.py +++ b/gnsq/message.py @@ -4,6 +4,8 @@ class Message(object): + """A class representing a message received from nsqd.""" + def __init__(self, conn, timestamp, attempts, id, body): self._has_responded = False self.conn = conn @@ -13,21 +15,31 @@ def __init__(self, conn, timestamp, attempts, id, body): self.body = body def has_responded(self): + """Returns whether or not this message has been responded to.""" return self._has_responded def finish(self): + """ + Respond to nsqd that you’ve processed this message successfully + (or would like to silently discard it). + """ if self._has_responded: raise NSQException('already responded') self._has_responded = True self.conn.finish(self.id) def requeue(self, time_ms=0): + """ + Respond to nsqd that you’ve failed to process this message successfully + (and would like it to be requeued). + """ if self._has_responded: raise NSQException('already responded') self._has_responded = True self.conn.requeue(self.id, time_ms) def touch(self): + """Respond to nsqd that you need more time to process the message.""" if self._has_responded: raise NSQException('already responded') self.conn.touch(self.id) From 47f0869f6fbaebf01b71a3b384db3c3dbe3c184f Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 16:46:10 -0700 Subject: [PATCH 078/113] Add lookupd docs. --- docs/lookupd.rst | 6 ++++++ gnsq/lookupd.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 docs/lookupd.rst diff --git a/docs/lookupd.rst b/docs/lookupd.rst new file mode 100644 index 0000000..e665ba1 --- /dev/null +++ b/docs/lookupd.rst @@ -0,0 +1,6 @@ +Nsqlookupd client +----------------- + +.. autoclass:: gnsq.Lookupd + :members: + :inherited-members: diff --git a/gnsq/lookupd.py b/gnsq/lookupd.py index 9e73a40..ce04451 100644 --- a/gnsq/lookupd.py +++ b/gnsq/lookupd.py @@ -5,6 +5,10 @@ class Lookupd(HTTPClient): + """Low level client for nsqlookupd. + + :param address: nsqlookupd http address (default: http://localhost:4161/) + """ def __init__(self, address='http://localhost:4161/'): if not address.endswith('/'): address += '/' @@ -12,6 +16,7 @@ def __init__(self, address='http://localhost:4161/'): self.address = self.base_url = address def lookup(self, topic): + """Returns producers for a topic.""" nsq.assert_valid_topic_name(topic) return self._json_api( self.url('lookup'), @@ -19,9 +24,11 @@ def lookup(self, topic): ) def topics(self): + """Returns all known topics.""" return self._json_api(self.url('topics')) def channels(self, topic): + """Returns all known channels of a topic.""" nsq.assert_valid_topic_name(topic) return self._json_api( self.url('channels'), @@ -29,9 +36,11 @@ def channels(self, topic): ) def nodes(self): + """Returns all known nsqd.""" return self._json_api(self.url('nodes')) def delete_topic(self, topic): + """Deletes an existing topic.""" nsq.assert_valid_topic_name(topic) return self._json_api( self.url('delete_topic'), @@ -39,6 +48,7 @@ def delete_topic(self, topic): ) def delete_channel(self, topic, channel): + """Deletes an existing channel of an existing topic.""" nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) return self._json_api( @@ -47,6 +57,7 @@ def delete_channel(self, topic, channel): ) def tombstone_topic_producer(self, topic, node): + """Tombstones a specific producer of an existing topic.""" nsq.assert_valid_topic_name(topic) return self._json_api( self.url('tombstone_topic_producer'), @@ -54,7 +65,12 @@ def tombstone_topic_producer(self, topic, node): ) def ping(self): + """Monitoring endpoint. + + :returns: should return `"OK"`, otherwise raises an exception. + """ return self._check_api(self.url('ping')) def info(self): + """Returns version information.""" return self._json_api(self.url('info')) From fb785aa512f03491cbae24e622f27358ddad55a5 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 16:46:26 -0700 Subject: [PATCH 079/113] Tweak nsqd section title. --- docs/nsqd.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/nsqd.rst b/docs/nsqd.rst index 7286a10..a189002 100644 --- a/docs/nsqd.rst +++ b/docs/nsqd.rst @@ -1,5 +1,5 @@ -Nsqd: individual nsqd connection ---------------- +Nsqd client +----------- .. autoclass:: gnsq.Nsqd :members: From d3fe2c6b7a9e2a3ab18ab2f0c400673358dc8ce5 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 17:03:42 -0700 Subject: [PATCH 080/113] Typo. --- gnsq/nsqd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index a4ceefa..63e3cbe 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -144,7 +144,7 @@ def __init__( self.user_agent = user_agent self.state = INIT - self.last_ressponse = time.time() + self.last_response = time.time() self.last_message = time.time() self.last_ready = 0 self.ready_count = 0 From 46a09b157c1bd979fca12be6282a6e779a03ceeb Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 17:04:16 -0700 Subject: [PATCH 081/113] Formatting fix. --- gnsq/reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 32ab829..4de7d93 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -34,7 +34,8 @@ class Reader(object): Messages will automatically be finished when the message handle returns unless the readers `async` flag is set to `True`. If an exception occurs or - :class:`gnsq.errors.NSQRequeueMessage` is raised, the message will be requeued. + :class:`gnsq.errors.NSQRequeueMessage` is raised, the message will be + requeued. The Reader will handle backing off of failed messages up to a configurable `max_interval` as well as automatically reconnecting to dropped connections. From 9a908be960181ca5d5ae9e1a385b1f27cec71aed Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 19:41:00 -0700 Subject: [PATCH 082/113] Fix english. --- gnsq/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 4de7d93..9366279 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -56,7 +56,7 @@ class Reader(object): .. data:: on_requeue(reader, message_id, timeout) :noindex: - Sent when a message is has been requeued. + Sent when a message is requeued. .. data:: on_giving_up(reader, message) :noindex: From 6779cd68c1edaaadf3a5f41c2474fda615b4f53c Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 20:17:06 -0700 Subject: [PATCH 083/113] Replace requests with lower level urllib3. --- gnsq/errors.py | 4 +++ gnsq/httpclient.py | 63 +++++++++++++++++++-------------- gnsq/lookupd.py | 38 +++++++------------- gnsq/nsqd.py | 70 ++++++++++++++----------------------- setup.py | 2 +- tests/integration_server.py | 9 ++--- 6 files changed, 85 insertions(+), 101 deletions(-) diff --git a/gnsq/errors.py b/gnsq/errors.py index 33c6e24..dd00220 100644 --- a/gnsq/errors.py +++ b/gnsq/errors.py @@ -15,6 +15,10 @@ class NSQNoConnections(NSQException): pass +class NSQHttpError(NSQException): + pass + + class NSQSocketError(socket.error, NSQException): pass diff --git a/gnsq/httpclient.py b/gnsq/httpclient.py index 7eae101..6013697 100644 --- a/gnsq/httpclient.py +++ b/gnsq/httpclient.py @@ -1,44 +1,53 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import requests -from .errors import NSQException + +import urllib3 + +try: + import simplejson as json +except ImportError: + import json # pyflakes.ignore + +from .errors import NSQHttpError class HTTPClient(object): base_url = None - _session = None + __http = None @property - def session(self): - if self._session is None: - self._session = requests.Session() - return self._session + def http(self): + if self.__http is None: + self.__http = urllib3.connection_from_url(url=self.base_url) + return self.__http - def url(self, *parts): - return self.base_url + '/'.join(parts) + def http_request(self, method, url, **kwargs): + response = self.http.request_encode_url(method, url, **kwargs) - def _check_connection(self): - pass + if 'application/json' in response.getheader('content-type', ''): + return self._http_check_json(response) - def _check_api(self, *args, **kwargs): - self._check_connection() + return self._http_check(response) - resp = self.session.post(*args, **kwargs) - if resp.status_code != 200: - raise NSQException(resp.status_code, 'api error') + def _http_check(self, response): + if response.status != 200: + raise NSQHttpError('http error <%s>' % response.status) + return response.data - return resp.text + def _http_check_json(self, response): + try: + data = json.loads(response.data) + except ValueError: + return self._http_check(response) - def _json_api(self, *args, **kwargs): - self._check_connection() + if response.status != 200: + status_txt = data.get('status_txt', 'http error') + raise NSQHttpError('%s <%s>' % (status_txt, response.status)) - resp = self.session.post(*args, **kwargs) - if resp.status_code != 200: - try: - msg = resp.json()['status_txt'] - except: - msg = 'api error' + return data['data'] - raise NSQException(resp.status_code, msg) + def http_get(self, url, **kwargs): + return self.http_request('GET', url, **kwargs) - return resp.json()['data'] + def http_post(self, url, **kwargs): + return self.http_request('POST', url, **kwargs) diff --git a/gnsq/lookupd.py b/gnsq/lookupd.py index ce04451..183612e 100644 --- a/gnsq/lookupd.py +++ b/gnsq/lookupd.py @@ -10,58 +10,46 @@ class Lookupd(HTTPClient): :param address: nsqlookupd http address (default: http://localhost:4161/) """ def __init__(self, address='http://localhost:4161/'): - if not address.endswith('/'): - address += '/' - self.address = self.base_url = address def lookup(self, topic): """Returns producers for a topic.""" nsq.assert_valid_topic_name(topic) - return self._json_api( - self.url('lookup'), - params={'topic': topic} - ) + return self.http_get('/lookup', fields={'topic': topic}) def topics(self): """Returns all known topics.""" - return self._json_api(self.url('topics')) + return self.http_get('/topics') def channels(self, topic): """Returns all known channels of a topic.""" nsq.assert_valid_topic_name(topic) - return self._json_api( - self.url('channels'), - params={'topic': topic} - ) + return self.http_get('/channels', fields={'topic': topic}) def nodes(self): """Returns all known nsqd.""" - return self._json_api(self.url('nodes')) + return self.http_get('/nodes') def delete_topic(self, topic): """Deletes an existing topic.""" nsq.assert_valid_topic_name(topic) - return self._json_api( - self.url('delete_topic'), - params={'topic': topic} - ) + return self.http_post('/delete_topic', fields={'topic': topic}) def delete_channel(self, topic, channel): """Deletes an existing channel of an existing topic.""" nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) - return self._json_api( - self.url('delete_channel'), - params={'topic': topic, 'channel': channel} + return self.http_post( + url='/delete_channel', + fields={'topic': topic, 'channel': channel}, ) def tombstone_topic_producer(self, topic, node): """Tombstones a specific producer of an existing topic.""" nsq.assert_valid_topic_name(topic) - return self._json_api( - self.url('tombstone_topic_producer'), - params={'topic': topic, 'node': node} + return self.http_post( + url='/tombstone_topic_producer', + fields={'topic': topic, 'node': node}, ) def ping(self): @@ -69,8 +57,8 @@ def ping(self): :returns: should return `"OK"`, otherwise raises an exception. """ - return self._check_api(self.url('ping')) + return self.http_get('/ping') def info(self): """Returns version information.""" - return self._json_api(self.url('info')) + return self.http_get('/info') diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 63e3cbe..c9899bc 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -376,19 +376,10 @@ def nop(self): def base_url(self): return 'http://%s:%s/' % (self.address, self.http_port) - def _check_connection(self): - if self.http_port: - return - raise errors.NSQException(-1, 'no http port') - def publish_http(self, topic, data): """Publish a message to the given topic over http.""" nsq.assert_valid_topic_name(topic) - return self._check_api( - self.url('put'), - params={'topic': topic}, - data=data - ) + return self.http_post('/put', fields={'topic': topic}, body=data) def multipublish_http(self, topic, messages): """Publish an iterable of messages to the given topic over http.""" @@ -401,61 +392,52 @@ def multipublish_http(self, topic, messages): error = 'newlines are not allowed in http multipublish' raise errors.NSQException(-1, error) - return self._check_api( - self.url('mput'), - params={'topic': topic}, - data='\n'.join(messages) + return self.http_post( + url='/mput', + fields={'topic': topic}, + body='\n'.join(messages) ) def create_topic(self, topic): """Create a topic.""" nsq.assert_valid_topic_name(topic) - return self._json_api( - self.url('create_topic'), - params={'topic': topic} - ) + return self.http_post('/create_topic', fields={'topic': topic}) def delete_topic(self, topic): """Delete a topic.""" nsq.assert_valid_topic_name(topic) - return self._json_api( - self.url('delete_topic'), - params={'topic': topic} - ) + return self.http_post('/delete_topic', fields={'topic': topic}) def create_channel(self, topic, channel): """Create a channel for an existing topic.""" nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) - return self._json_api( - self.url('create_channel'), - params={'topic': topic, 'channel': channel} + return self.http_post( + url='/create_channel', + fields={'topic': topic, 'channel': channel}, ) def delete_channel(self, topic, channel): """Delete an existing channel for an existing topic.""" nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) - return self._json_api( - self.url('delete_channel'), - params={'topic': topic, 'channel': channel} + return self.http_post( + url='/delete_channel', + fields={'topic': topic, 'channel': channel}, ) def empty_topic(self, topic): """Empty all the queued messages for an existing topic.""" nsq.assert_valid_topic_name(topic) - return self._json_api( - self.url('empty_topic'), - params={'topic': topic} - ) + return self.http_post('/empty_topic', fields={'topic': topic}) def empty_channel(self, topic, channel): """Empty all the queued messages for an existing channel.""" nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) - return self._json_api( - self.url('empty_channel'), - params={'topic': topic, 'channel': channel} + return self.http_post( + url='/empty_channel', + fields={'topic': topic, 'channel': channel}, ) def pause_channel(self, topic, channel): @@ -465,34 +447,34 @@ def pause_channel(self, topic, channel): """ nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) - return self._json_api( - self.url('pause_channel'), - params={'topic': topic, 'channel': channel} + return self.http_post( + url='/pause_channel', + fields={'topic': topic, 'channel': channel}, ) def unpause_channel(self, topic, channel): """Resume message flow to channels of an existing, paused, topic.""" nsq.assert_valid_topic_name(topic) nsq.assert_valid_channel_name(channel) - return self._json_api( - self.url('unpause_channel'), - params={'topic': topic, 'channel': channel} + return self.http_post( + url='/unpause_channel', + fields={'topic': topic, 'channel': channel}, ) def stats(self): """Return internal instrumented statistics.""" - return self._json_api(self.url('stats'), params={'format': 'json'}) + return self.http_get('/stats', fields={'format': 'json'}) def ping(self): """Monitoring endpoint. :returns: should return `"OK"`, otherwise raises an exception. """ - return self._check_api(self.url('ping')) + return self.http_get('/ping') def info(self): """Returns version information.""" - return self._json_api(self.url('info')) + return self.http_get('/info') def publish(self, topic, data): """Publish a message. diff --git a/setup.py b/setup.py index e10f584..5126b1b 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ install_requires=[ 'gevent', 'blinker', - 'requests', + 'urllib3', ], license="BSD", zip_safe=False, diff --git a/tests/integration_server.py b/tests/integration_server.py index 924d509..ae31d13 100644 --- a/tests/integration_server.py +++ b/tests/integration_server.py @@ -3,13 +3,14 @@ import shutil import subprocess import tempfile -import requests import os.path +import urllib3 class IntegrationNsqdServer(object): tls_cert = os.path.join(os.path.dirname(__file__), 'cert.pem') tls_key = os.path.join(os.path.dirname(__file__), 'key.pem') + http = urllib3.PoolManager() def __init__(self, address=None, tcp_port=None, http_port=None): if address is None: @@ -37,9 +38,9 @@ def http_address(self): def is_running(self): try: - resp = requests.get('http://%s/ping' % self.http_address) - return resp.text == 'OK' - except requests.ConnectionError: + url = 'http://%s/ping' % self.http_address + return self.http.request('GET', url).data == 'OK' + except urllib3.exceptions.HTTPError: return False def wait(self): From f17756471ca6fba20d7f3230aa2eda722a038659 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 20:17:16 -0700 Subject: [PATCH 084/113] Words. --- gnsq/nsqd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index c9899bc..149639b 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -32,7 +32,7 @@ class Nsqd(HTTPClient): .. data:: on_message(conn, message) :noindex: - Sent when the connections receives a message. + Sent when a message frame is received. .. data:: on_finish(conn, message_id) :noindex: @@ -47,17 +47,17 @@ class Nsqd(HTTPClient): .. data:: on_response(conn, response) :noindex: - Sent when the connections receives a response. + Sent when a response frame is received. .. data:: on_error(conn, error) :noindex: - Sent when the connections receives an error frame. + Sent when an error frame is received. .. data:: on_close(conn) :noindex: - Sent when the connections stream is closed. + Sent when the stream is closed. :param address: the host or ip address of the nsqd From ad71258d5657b69a8ace2a9af48b82a349ddfc90 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 20:50:12 -0700 Subject: [PATCH 085/113] Add nsqd http tests. --- tests/test_nsqd_http.py | 74 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/test_nsqd_http.py diff --git a/tests/test_nsqd_http.py b/tests/test_nsqd_http.py new file mode 100644 index 0000000..8c5f9a9 --- /dev/null +++ b/tests/test_nsqd_http.py @@ -0,0 +1,74 @@ +from __future__ import with_statement + +import pytest +import gnsq +from integration_server import IntegrationNsqdServer + + +@pytest.mark.slow +def test_basic(): + with IntegrationNsqdServer() as server: + conn = gnsq.Nsqd(server.address, http_port=server.http_port) + assert conn.ping() == 'OK' + assert 'topics' in conn.stats() + assert 'version' in conn.info() + + +@pytest.mark.slow +def test_topics_channels(): + with IntegrationNsqdServer() as server: + conn = gnsq.Nsqd(server.address, http_port=server.http_port) + assert len(conn.stats()['topics']) == 0 + + with pytest.raises(gnsq.errors.NSQHttpError): + conn.delete_topic('topic') + + conn.create_topic('topic') + topics = conn.stats()['topics'] + assert len(topics) == 1 + assert topics[0]['topic_name'] == 'topic' + + conn.delete_topic('topic') + assert len(conn.stats()['topics']) == 0 + + with pytest.raises(gnsq.errors.NSQHttpError): + conn.create_channel('topic', 'channel') + + with pytest.raises(gnsq.errors.NSQHttpError): + conn.delete_channel('topic', 'channel') + + conn.create_topic('topic') + assert len(conn.stats()['topics'][0]['channels']) == 0 + + conn.create_channel('topic', 'channel') + channels = conn.stats()['topics'][0]['channels'] + assert len(channels) == 1 + assert channels[0]['channel_name'] == 'channel' + + conn.delete_channel('topic', 'channel') + assert len(conn.stats()['topics'][0]['channels']) == 0 + + +def test_publish(): + with IntegrationNsqdServer() as server: + conn = gnsq.Nsqd(server.address, http_port=server.http_port) + + conn.publish_http('topic', 'sup') + assert conn.stats()['topics'][0]['depth'] == 1 + + conn.multipublish_http('topic', ['sup', 'sup']) + assert conn.stats()['topics'][0]['depth'] == 3 + + conn.multipublish_http('topic', iter(['sup', 'sup', 'sup'])) + assert conn.stats()['topics'][0]['depth'] == 6 + + conn.empty_topic('topic') + assert conn.stats()['topics'][0]['depth'] == 0 + + conn.create_topic('topic') + conn.create_channel('topic', 'channel') + conn.publish_http('topic', 'sup') + assert conn.stats()['topics'][0]['channels'][0]['depth'] == 1 + + conn.empty_channel('topic', 'channel') + assert conn.stats()['topics'][0]['channels'][0]['depth'] == 0 From 81db325e3edd795b62bf52379063aa52ccb71da2 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 20:50:55 -0700 Subject: [PATCH 086/113] Fix Nsqd:multipublish to work with iterators. --- gnsq/nsqd.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 149639b..63631e2 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -381,21 +381,20 @@ def publish_http(self, topic, data): nsq.assert_valid_topic_name(topic) return self.http_post('/put', fields={'topic': topic}, body=data) + def _validate_http_mpub(self, message): + if '\n' not in message: + return message + + error = 'newlines are not allowed in http multipublish' + raise errors.NSQException(error) + def multipublish_http(self, topic, messages): """Publish an iterable of messages to the given topic over http.""" nsq.assert_valid_topic_name(topic) - - for message in messages: - if '\n' not in message: - continue - - error = 'newlines are not allowed in http multipublish' - raise errors.NSQException(-1, error) - return self.http_post( url='/mput', fields={'topic': topic}, - body='\n'.join(messages) + body='\n'.join(self._validate_http_mpub(m) for m in messages) ) def create_topic(self, topic): From dcf72b6b127c5fbe147aa1a6695ce761268f0c17 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 21:17:56 -0700 Subject: [PATCH 087/113] Add lookupd integration server. --- tests/integration_server.py | 42 ++++++++++++++++++++++++++----------- tests/test_nsqd.py | 8 +++---- tests/test_nsqd_http.py | 8 +++---- tests/test_reader.py | 4 ++-- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/tests/integration_server.py b/tests/integration_server.py index ae31d13..424c482 100644 --- a/tests/integration_server.py +++ b/tests/integration_server.py @@ -7,9 +7,7 @@ import urllib3 -class IntegrationNsqdServer(object): - tls_cert = os.path.join(os.path.dirname(__file__), 'cert.pem') - tls_key = os.path.join(os.path.dirname(__file__), 'key.pem') +class BaseIntegrationServer(object): http = urllib3.PoolManager() def __init__(self, address=None, tcp_port=None, http_port=None): @@ -28,6 +26,10 @@ def __init__(self, address=None, tcp_port=None, http_port=None): self.http_port = http_port self.data_path = tempfile.mkdtemp() + @property + def cmd(self): + raise NotImplementedError('cmd not implemented') + @property def tcp_address(self): return '%s:%d' % (self.address, self.tcp_port) @@ -48,8 +50,24 @@ def wait(self): if self.is_running(): return time.sleep(0.01 * pow(2, attempt)) - raise RuntimeError('unable to start nsqd') + raise RuntimeError('unable to start: %r' % ' '.join(self.cmd)) + + def __enter__(self): + self.subp = subprocess.Popen(self.cmd) + self.wait() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.subp.terminate() + self.subp.wait() + shutil.rmtree(self.data_path) + + +class NsqdIntegrationServer(BaseIntegrationServer): + tls_cert = os.path.join(os.path.dirname(__file__), 'cert.pem') + tls_key = os.path.join(os.path.dirname(__file__), 'key.pem') + @property def cmd(self): return [ 'nsqd', @@ -60,12 +78,12 @@ def cmd(self): '--tls-key', self.tls_key, ] - def __enter__(self): - self.nsqd = subprocess.Popen(self.cmd()) - self.wait() - return self - def __exit__(self, exc_type, exc_value, traceback): - self.nsqd.terminate() - self.nsqd.wait() - shutil.rmtree(self.data_path) +class LookupdIntegrationServer(BaseIntegrationServer): + @property + def cmd(self): + return [ + 'nsqlookupd', + '--tcp-address', self.tcp_address, + '--http-address', self.http_address, + ] diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index 0a01d15..dd6f26c 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -9,7 +9,7 @@ from gnsq.stream.stream import SSLSocket, DefalteSocket, SnappySocket from mock_server import mock_server -from integration_server import IntegrationNsqdServer +from integration_server import NsqdIntegrationServer def mock_response(frame_type, data): @@ -218,7 +218,7 @@ def handle(socket, address): @pytest.mark.slow def test_tls(): - with IntegrationNsqdServer() as server: + with NsqdIntegrationServer() as server: conn = Nsqd( address=server.address, tcp_port=server.tcp_port, @@ -243,7 +243,7 @@ def test_tls(): @pytest.mark.slow def test_deflate(): - with IntegrationNsqdServer() as server: + with NsqdIntegrationServer() as server: conn = Nsqd( address=server.address, tcp_port=server.tcp_port, @@ -264,7 +264,7 @@ def test_deflate(): @pytest.mark.slow def test_snappy(): - with IntegrationNsqdServer() as server: + with NsqdIntegrationServer() as server: conn = Nsqd( address=server.address, tcp_port=server.tcp_port, diff --git a/tests/test_nsqd_http.py b/tests/test_nsqd_http.py index 8c5f9a9..cb288c4 100644 --- a/tests/test_nsqd_http.py +++ b/tests/test_nsqd_http.py @@ -2,12 +2,12 @@ import pytest import gnsq -from integration_server import IntegrationNsqdServer +from integration_server import NsqdIntegrationServer @pytest.mark.slow def test_basic(): - with IntegrationNsqdServer() as server: + with NsqdIntegrationServer() as server: conn = gnsq.Nsqd(server.address, http_port=server.http_port) assert conn.ping() == 'OK' assert 'topics' in conn.stats() @@ -16,7 +16,7 @@ def test_basic(): @pytest.mark.slow def test_topics_channels(): - with IntegrationNsqdServer() as server: + with NsqdIntegrationServer() as server: conn = gnsq.Nsqd(server.address, http_port=server.http_port) assert len(conn.stats()['topics']) == 0 @@ -50,7 +50,7 @@ def test_topics_channels(): def test_publish(): - with IntegrationNsqdServer() as server: + with NsqdIntegrationServer() as server: conn = gnsq.Nsqd(server.address, http_port=server.http_port) conn.publish_http('topic', 'sup') diff --git a/tests/test_reader.py b/tests/test_reader.py index 8dfbeea..abf038b 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -1,12 +1,12 @@ import pytest from gnsq import Nsqd, Reader from gnsq.errors import NSQSocketError -from integration_server import IntegrationNsqdServer +from integration_server import NsqdIntegrationServer @pytest.mark.slow def test_messages(): - with IntegrationNsqdServer() as server: + with NsqdIntegrationServer() as server: class Accounting(object): count = 0 From 5b8bfba1471af67fc7a89a645653ceaa77c6db4b Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 21:18:12 -0700 Subject: [PATCH 088/113] Basic lookupd tests. --- tests/test_lookupd.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/test_lookupd.py diff --git a/tests/test_lookupd.py b/tests/test_lookupd.py new file mode 100644 index 0000000..c350f1f --- /dev/null +++ b/tests/test_lookupd.py @@ -0,0 +1,13 @@ +from __future__ import with_statement + +import pytest +import gnsq +from integration_server import LookupdIntegrationServer + + +@pytest.mark.slow +def test_basic(): + with LookupdIntegrationServer() as server: + conn = gnsq.Lookupd(server.http_address) + assert conn.ping() == 'OK' + assert 'version' in conn.info() From c59d04814a66fea413633b38266ef4203cb1d60d Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 21:18:30 -0700 Subject: [PATCH 089/113] Fix coverage makefile task. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e104457..cc2baa4 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ test-all: tox coverage: - coverage run --source gnsq py.test + py.test --runslow --cov gnsq tests coverage report -m coverage html open htmlcov/index.html From 3d287e7863898bc44b4edde0dc1699366b84c1ec Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 21:21:48 -0700 Subject: [PATCH 090/113] Test backofftimer reset and min_interval. --- tests/test_basic.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_basic.py b/tests/test_basic.py index 1caef8c..bfa467a 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -59,4 +59,12 @@ def test_backoff_timer(): timer.failure() assert timer.c == 100 + assert timer.get_interval() <= 1000 + + timer.reset() + assert timer.c == 0 + assert timer.get_interval() == 0 + + timer = BackoffTimer(min_interval=1000) + assert timer.c == 0 assert timer.get_interval() == 1000 From 4e1aac8fc64cd2cda83e3b499770c31f5fdd3313 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 21:58:38 -0700 Subject: [PATCH 091/113] Use random port. --- tests/integration_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration_server.py b/tests/integration_server.py index 424c482..4f35349 100644 --- a/tests/integration_server.py +++ b/tests/integration_server.py @@ -16,7 +16,6 @@ def __init__(self, address=None, tcp_port=None, http_port=None): if tcp_port is None: tcp_port = random.randint(10000, 65535) - tcp_port = 1234 if http_port is None: http_port = tcp_port + 1 From d4c65f903a5967f3012222dfa9eda4d140661f1a Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 21:59:30 -0700 Subject: [PATCH 092/113] Allow configurable lookupd for NsqdIntegrationServer. --- tests/integration_server.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/integration_server.py b/tests/integration_server.py index 4f35349..9e27284 100644 --- a/tests/integration_server.py +++ b/tests/integration_server.py @@ -66,9 +66,13 @@ class NsqdIntegrationServer(BaseIntegrationServer): tls_cert = os.path.join(os.path.dirname(__file__), 'cert.pem') tls_key = os.path.join(os.path.dirname(__file__), 'key.pem') + def __init__(self, lookupd=None, **kwargs): + self.lookupd = lookupd + super(NsqdIntegrationServer, self).__init__(**kwargs) + @property def cmd(self): - return [ + cmd = [ 'nsqd', '--tcp-address', self.tcp_address, '--http-address', self.http_address, @@ -77,6 +81,11 @@ def cmd(self): '--tls-key', self.tls_key, ] + if self.lookupd: + cmd.extend(['--lookupd-tcp-address', self.lookupd]) + + return cmd + class LookupdIntegrationServer(BaseIntegrationServer): @property From aa25439240e1e5597244f2e7f764d87b0b81be4c Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 22:05:53 -0700 Subject: [PATCH 093/113] Lookupd integration tests. --- tests/test_lookupd.py | 44 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/test_lookupd.py b/tests/test_lookupd.py index c350f1f..ca53898 100644 --- a/tests/test_lookupd.py +++ b/tests/test_lookupd.py @@ -2,12 +2,48 @@ import pytest import gnsq -from integration_server import LookupdIntegrationServer + +from integration_server import LookupdIntegrationServer, NsqdIntegrationServer @pytest.mark.slow def test_basic(): with LookupdIntegrationServer() as server: - conn = gnsq.Lookupd(server.http_address) - assert conn.ping() == 'OK' - assert 'version' in conn.info() + lookupd = gnsq.Lookupd(server.http_address) + assert lookupd.ping() == 'OK' + assert 'version' in lookupd.info() + + with pytest.raises(gnsq.errors.NSQHttpError): + lookupd.lookup('topic') + + assert len(lookupd.topics()['topics']) == 0 + assert len(lookupd.channels('topic')['channels']) == 0 + assert len(lookupd.nodes()['producers']) == 0 + + +@pytest.mark.slow +def test_lookup(): + lookupd_server = LookupdIntegrationServer() + nsqd_server = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) + + with lookupd_server, nsqd_server: + lookupd = gnsq.Lookupd(lookupd_server.http_address) + conn = gnsq.Nsqd(nsqd_server.address, http_port=nsqd_server.http_port) + + assert len(lookupd.topics()['topics']) == 0 + assert len(lookupd.channels('topic')['channels']) == 0 + assert len(lookupd.nodes()['producers']) == 1 + + conn.create_topic('topic') + info = lookupd.lookup('topic') + assert len(info['channels']) == 0 + assert len(info['producers']) == 1 + assert len(lookupd.topics()['topics']) == 1 + assert len(lookupd.channels('topic')['channels']) == 0 + + conn.create_channel('topic', 'channel') + info = lookupd.lookup('topic') + assert len(info['channels']) == 1 + assert len(info['producers']) == 1 + assert len(lookupd.topics()['topics']) == 1 + assert len(lookupd.channels('topic')['channels']) == 1 From 35bd9d2de64ba0e86e62e7c969d2e5f2f05aa7fc Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 22:24:11 -0700 Subject: [PATCH 094/113] Message tests. --- tests/test_message.py | 101 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tests/test_message.py diff --git a/tests/test_message.py b/tests/test_message.py new file mode 100644 index 0000000..b894658 --- /dev/null +++ b/tests/test_message.py @@ -0,0 +1,101 @@ +import pytest +import gnsq + + +class MockConnection(object): + def __init__(self, operations): + self.operations = iter(operations) + + def __getattr__(self, name): + expected_name, expected_args = self.operations.next() + assert name == expected_name + + def check_args(*args): + assert args == expected_args + + return check_args + + def assert_finished(self): + with pytest.raises(StopIteration): + self.operations.next() + + +def test_basic(): + message = gnsq.Message(None, 0, 42, '1234', 'sup') + assert message.id == '1234' + assert message.body == 'sup' + assert message.has_responded() is False + + +def test_finish(): + mock_conn = MockConnection([ + ('finish', ('1234',)), + ]) + + message = gnsq.Message(mock_conn, 0, 42, '1234', 'sup') + assert message.has_responded() is False + + message.finish() + assert message.has_responded() is True + + with pytest.raises(gnsq.errors.NSQException): + message.finish() + + +def test_requeue(): + mock_conn = MockConnection([ + ('requeue', ('1234', 0)), + ]) + + message = gnsq.Message(mock_conn, 0, 42, '1234', 'sup') + assert message.has_responded() is False + + message.requeue() + assert message.has_responded() is True + + with pytest.raises(gnsq.errors.NSQException): + message.requeue() + + mock_conn.assert_finished() + + +def test_requeue_timeout(): + mock_conn = MockConnection([ + ('requeue', ('1234', 1000)), + ]) + + message = gnsq.Message(mock_conn, 0, 42, '1234', 'sup') + assert message.has_responded() is False + + message.requeue(1000) + assert message.has_responded() is True + + with pytest.raises(gnsq.errors.NSQException): + message.requeue(1000) + + mock_conn.assert_finished() + + +def test_touch(): + mock_conn = MockConnection([ + ('touch', ('1234',)), + ('touch', ('1234',)), + ('touch', ('1234',)), + ('finish', ('1234',)), + ]) + + message = gnsq.Message(mock_conn, 0, 42, '1234', 'sup') + assert message.has_responded() is False + + message.touch() + message.touch() + message.touch() + assert message.has_responded() is False + + message.finish() + assert message.has_responded() is True + + with pytest.raises(gnsq.errors.NSQException): + message.touch() + + mock_conn.assert_finished() From 5cb8b0dc9eebe3bc9684dbb2e20367316534ad84 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 22:51:08 -0700 Subject: [PATCH 095/113] Check unconnected defaults to http. --- tests/test_nsqd_http.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_nsqd_http.py b/tests/test_nsqd_http.py index cb288c4..69b3c3d 100644 --- a/tests/test_nsqd_http.py +++ b/tests/test_nsqd_http.py @@ -53,13 +53,13 @@ def test_publish(): with NsqdIntegrationServer() as server: conn = gnsq.Nsqd(server.address, http_port=server.http_port) - conn.publish_http('topic', 'sup') + conn.publish('topic', 'sup') assert conn.stats()['topics'][0]['depth'] == 1 - conn.multipublish_http('topic', ['sup', 'sup']) + conn.multipublish('topic', ['sup', 'sup']) assert conn.stats()['topics'][0]['depth'] == 3 - conn.multipublish_http('topic', iter(['sup', 'sup', 'sup'])) + conn.multipublish('topic', iter(['sup', 'sup', 'sup'])) assert conn.stats()['topics'][0]['depth'] == 6 conn.empty_topic('topic') @@ -67,7 +67,7 @@ def test_publish(): conn.create_topic('topic') conn.create_channel('topic', 'channel') - conn.publish_http('topic', 'sup') + conn.publish('topic', 'sup') assert conn.stats()['topics'][0]['channels'][0]['depth'] == 1 conn.empty_channel('topic', 'channel') From ec83d21669d217606f21c1782de2cee47982048e Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 22:51:38 -0700 Subject: [PATCH 096/113] Test unconnected nsqd, publishing and heartbeats. --- tests/test_nsqd.py | 82 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index dd6f26c..224296e 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -40,10 +40,32 @@ def handle(socket, address): conn.connect() assert conn.state == states.CONNECTED + conn.connect() + assert conn.state == states.CONNECTED + conn.close_stream() assert conn.state == states.DISCONNECTED +def test_disconnected(): + @mock_server + def handle(socket, address): + assert socket.recv(4) == ' V2' + assert socket.recv(1) == '' + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + conn.connect() + conn.close_stream() + assert conn.state == states.DISCONNECTED + + with pytest.raises(errors.NSQSocketError): + conn.nop() + + with pytest.raises(errors.NSQSocketError): + conn.read_response() + + @pytest.mark.parametrize('body', [ 'hello world', '', @@ -130,6 +152,48 @@ def handle(socket, address): getattr(conn, command)(*args) +def test_publish(): + @mock_server + def handle(socket, address): + assert socket.recv(4) == ' V2' + assert socket.recv(10) == 'PUB topic\n' + + assert nsq.unpack_size(socket.recv(4)) == 3 + assert socket.recv(3) == 'sup' + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + conn.connect() + conn.publish('topic', 'sup') + + +def test_multipublish(): + @mock_server + def handle(socket, address): + assert socket.recv(4) == ' V2' + assert socket.recv(11) == 'MPUB topic\n' + + size = nsq.unpack_size(socket.recv(4)) + data = socket.recv(size) + + head, data = data[:4], data[4:] + assert nsq.unpack_size(head) == 2 + + for _ in xrange(2): + head, data = data[:4], data[4:] + assert nsq.unpack_size(head) == 3 + + head, data = data[:3], data[3:] + assert head == 'sup' + + assert data == '' + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + conn.connect() + conn.multipublish('topic', ['sup', 'sup']) + + @pytest.mark.parametrize('error,error_class,fatal', [ ('E_INVALID', errors.NSQInvalid, True), ('E_BAD_BODY', errors.NSQBadBody, True), @@ -164,6 +228,8 @@ def test_hashing(): conn1 = Nsqd('localhost', 1337) conn2 = Nsqd('localhost', 1337) assert conn1 == conn2 + assert not (conn1 < conn2) + assert not (conn2 < conn1) test = {conn1: True} assert conn2 in test @@ -216,6 +282,22 @@ def handle(socket, address): assert json.loads(msg.body)['data']['test_key'] == i +def test_sync_heartbeat(): + @mock_server + def handle(socket, address): + assert socket.recv(4) == ' V2' + socket.send(mock_response(nsq.FRAME_TYPE_RESPONSE, '_heartbeat_')) + assert socket.recv(4) == 'NOP\n' + + with handle as server: + conn = Nsqd(address='127.0.0.1', tcp_port=server.server_port) + conn.connect() + + frame, data = conn.read_response() + assert frame == nsq.FRAME_TYPE_RESPONSE + assert data == '_heartbeat_' + + @pytest.mark.slow def test_tls(): with NsqdIntegrationServer() as server: From 9e67d8629f8c0f0b344b36fac5f14d0242324f07 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 22:59:36 -0700 Subject: [PATCH 097/113] Test protocol edge cases. --- tests/test_basic.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_basic.py b/tests/test_basic.py index bfa467a..0a4b9a6 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -27,6 +27,31 @@ def test_channel_names(name, good): assert nsq.valid_channel_name(name) == good +def test_assert_topic(): + assert nsq.assert_valid_topic_name('topic') is None + + with pytest.raises(ValueError): + nsq.assert_valid_topic_name('invalid name with space') + + +def test_assert_channel(): + assert nsq.assert_valid_channel_name('channel') is None + + with pytest.raises(ValueError): + nsq.assert_valid_channel_name('invalid name with space') + + +def test_invalid_commands(): + with pytest.raises(TypeError): + nsq.requeue('1234', None) + + with pytest.raises(TypeError): + nsq.ready(None) + + with pytest.raises(ValueError): + nsq.ready(-1) + + def test_backoff_timer(): timer = BackoffTimer(max_interval=1000) assert timer.get_interval() == 0 From d477a94e066d4b567ec9679386fdb2ccbcf9054d Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sat, 5 Jul 2014 23:56:59 -0700 Subject: [PATCH 098/113] Fix error decoding. --- gnsq/errors.py | 3 ++- tests/test_nsqd.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/gnsq/errors.py b/gnsq/errors.py index dd00220..8e9b91d 100644 --- a/gnsq/errors.py +++ b/gnsq/errors.py @@ -102,4 +102,5 @@ class NSQTouchFailed(NSQErrorCode): def make_error(error_code): - return ERROR_CODES.get(error_code, NSQErrorCode)(error_code) + parts = error_code.split(None, 1) + return ERROR_CODES.get(parts[0], NSQErrorCode)(parts[-1]) diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index 224296e..15e1302 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -322,6 +322,11 @@ def test_tls(): assert frame == nsq.FRAME_TYPE_RESPONSE assert data == 'OK' + conn.publish('topic', 'sup') + frame, data = conn.read_response() + assert frame == nsq.FRAME_TYPE_RESPONSE + assert data == 'OK' + @pytest.mark.slow def test_deflate(): @@ -363,3 +368,17 @@ def test_snappy(): frame, data = conn.read_response() assert frame == nsq.FRAME_TYPE_RESPONSE assert data == 'OK' + + +@pytest.mark.slow +def test_cls_error(): + with NsqdIntegrationServer() as server: + conn = Nsqd(address=server.address, tcp_port=server.tcp_port) + + conn.connect() + assert conn.state == states.CONNECTED + + conn.close() + frame, error = conn.read_response() + assert frame == nsq.FRAME_TYPE_ERROR + assert isinstance(error, errors.NSQInvalid) From d1cc24ed1fe5f7e6fc1fa7f658f584d080257ec2 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 6 Jul 2014 00:02:24 -0700 Subject: [PATCH 099/113] Cleanup compression api. --- gnsq/stream/compression.py | 6 ------ tests/test_nsqd.py | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gnsq/stream/compression.py b/gnsq/stream/compression.py index f756786..89149f8 100644 --- a/gnsq/stream/compression.py +++ b/gnsq/stream/compression.py @@ -17,12 +17,6 @@ def bootstrap(self, data): return self._bootstrapped = self.decompress(data) - def compress(self, data): - return data - - def decompress(self, data): - return data - def recv(self, size): if self._bootstrapped: data = self._bootstrapped diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index 15e1302..31d46d2 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -327,6 +327,8 @@ def test_tls(): assert frame == nsq.FRAME_TYPE_RESPONSE assert data == 'OK' + conn.close() + @pytest.mark.slow def test_deflate(): @@ -348,6 +350,8 @@ def test_deflate(): assert frame == nsq.FRAME_TYPE_RESPONSE assert data == 'OK' + conn.close() + @pytest.mark.slow def test_snappy(): @@ -369,6 +373,8 @@ def test_snappy(): assert frame == nsq.FRAME_TYPE_RESPONSE assert data == 'OK' + conn.close() + @pytest.mark.slow def test_cls_error(): From 78a258b2eee4ca634a57fe8a093588067362e3be Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 6 Jul 2014 01:09:09 -0700 Subject: [PATCH 100/113] Configure https port for integration server. --- tests/integration_server.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/integration_server.py b/tests/integration_server.py index 9e27284..3bc55b5 100644 --- a/tests/integration_server.py +++ b/tests/integration_server.py @@ -70,12 +70,21 @@ def __init__(self, lookupd=None, **kwargs): self.lookupd = lookupd super(NsqdIntegrationServer, self).__init__(**kwargs) + @property + def https_port(self): + return self.http_port + 1 + + @property + def https_address(self): + return '%s:%d' % (self.address, self.https_port) + @property def cmd(self): cmd = [ 'nsqd', '--tcp-address', self.tcp_address, '--http-address', self.http_address, + '--https-address', self.https_address, '--data-path', self.data_path, '--tls-cert', self.tls_cert, '--tls-key', self.tls_key, From 3ef5006ef9238ae264fff3479c5619794eca854f Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 6 Jul 2014 01:10:16 -0700 Subject: [PATCH 101/113] Test reader basic functions, max concurrency, lookupd and back off. --- tests/test_reader.py | 204 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 202 insertions(+), 2 deletions(-) diff --git a/tests/test_reader.py b/tests/test_reader.py index abf038b..11eeb75 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -1,7 +1,57 @@ +from __future__ import with_statement + +import multiprocessing import pytest -from gnsq import Nsqd, Reader +import gevent + +from gnsq import Nsqd, Reader, states from gnsq.errors import NSQSocketError -from integration_server import NsqdIntegrationServer + +from integration_server import NsqdIntegrationServer, LookupdIntegrationServer + + +def test_basic(): + with pytest.raises(ValueError): + Reader('test', 'test') + + with pytest.raises(TypeError): + Reader( + topic='test', + channel='test', + nsqd_tcp_addresses=None, + lookupd_http_addresses='http://localhost:4161/', + ) + + with pytest.raises(TypeError): + Reader( + topic='test', + channel='test', + nsqd_tcp_addresses='localhost:4150', + lookupd_http_addresses=None, + ) + + def message_handler(reader, message): + pass + + reader = Reader( + topic='test', + channel='test', + name='test', + max_concurrency=-1, + nsqd_tcp_addresses='localhost:4150', + lookupd_http_addresses='http://localhost:4161/', + message_handler=message_handler + ) + + assert reader.name == 'test' + assert reader.max_concurrency == multiprocessing.cpu_count() + assert len(reader.on_message.receivers) == 1 + + assert isinstance(reader.nsqd_tcp_addresses, set) + assert len(reader.nsqd_tcp_addresses) == 1 + + assert isinstance(reader.lookupds, list) + assert len(reader.lookupds) == 1 @pytest.mark.slow @@ -40,6 +90,71 @@ def error_handler(reader, message, error): def handler(reader, message): assert message.body == 'danger zone!' + Accounting.count += 1 + if Accounting.count == Accounting.total: + assert not reader.is_starved + reader.close() + + try: + reader.start() + except NSQSocketError: + pass + + if Accounting.error: + raise Accounting.error + + assert Accounting.count == Accounting.total + + +@pytest.mark.slow +def test_max_concurrency(): + server1 = NsqdIntegrationServer() + server2 = NsqdIntegrationServer() + + with server1, server2: + class Accounting(object): + count = 0 + total = 100 + concurrency = 0 + error = None + + for server in (server1, server2): + conn = Nsqd( + address=server.address, + tcp_port=server.tcp_port, + http_port=server.http_port, + ) + + for _ in xrange(Accounting.total / 2): + conn.publish_http('test', 'danger zone!') + + reader = Reader( + topic='test', + channel='test', + nsqd_tcp_addresses=[ + server1.tcp_address, + server2.tcp_address, + ], + max_in_flight=5, + max_concurrency=1, + ) + + @reader.on_exception.connect + def error_handler(reader, message, error): + if isinstance(error, NSQSocketError): + return + Accounting.error = error + reader.close() + + @reader.on_message.connect + def handler(reader, message): + assert message.body == 'danger zone!' + assert Accounting.concurrency == 0 + + Accounting.concurrency += 1 + gevent.sleep() + Accounting.concurrency -= 1 + Accounting.count += 1 if Accounting.count == Accounting.total: reader.close() @@ -53,3 +168,88 @@ def handler(reader, message): raise Accounting.error assert Accounting.count == Accounting.total + + +@pytest.mark.slow +def test_lookupd(): + lookupd_server = LookupdIntegrationServer() + server1 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) + server2 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) + + with lookupd_server, server1, server2: + class Accounting(object): + count = 0 + total = 500 + concurrency = 0 + error = None + + for server in (server1, server2): + conn = Nsqd( + address=server.address, + tcp_port=server.tcp_port, + http_port=server.http_port, + ) + + for _ in xrange(Accounting.total / 2): + conn.publish_http('test', 'danger zone!') + + reader = Reader( + topic='test', + channel='test', + lookupd_http_addresses=lookupd_server.http_address, + max_in_flight=32, + ) + + @reader.on_exception.connect + def error_handler(reader, message, error): + if isinstance(error, NSQSocketError): + return + Accounting.error = error + reader.close() + + @reader.on_message.connect + def handler(reader, message): + assert message.body == 'danger zone!' + + Accounting.count += 1 + if Accounting.count == Accounting.total: + reader.close() + + try: + reader.start() + except NSQSocketError: + pass + + if Accounting.error: + raise Accounting.error + + assert Accounting.count == Accounting.total + + +@pytest.mark.slow +def test_backoff(): + with NsqdIntegrationServer() as server: + conn = Nsqd( + address=server.address, + tcp_port=server.tcp_port, + http_port=server.http_port, + ) + + for _ in xrange(500): + conn.publish_http('test', 'danger zone!') + + reader = Reader( + topic='test', + channel='test', + nsqd_tcp_addresses=[server.tcp_address], + max_in_flight=100, + ) + + reader.start(block=False) + reader.start_backoff() + + assert reader.state == states.THROTTLED + assert reader.total_in_flight_or_ready <= 1 + + reader.complete_backoff() + assert reader.state == states.RUNNING From 52f3eaf5368ecd026e44287777d4de5563f354c4 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 6 Jul 2014 01:16:24 -0700 Subject: [PATCH 102/113] Fix setting max concurrency. --- gnsq/reader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 9366279..47f363e 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -168,7 +168,6 @@ def __init__( self.async = async self.max_tries = max_tries self.max_in_flight = max_in_flight - self.max_concurrency = max_concurrency self.requeue_delay = requeue_delay self.lookupd_poll_interval = lookupd_poll_interval self.lookupd_poll_jitter = lookupd_poll_jitter @@ -201,7 +200,9 @@ def __init__( self.on_message.connect(message_handler) if max_concurrency < 0: - max_concurrency = cpu_count() + self.max_concurrency = cpu_count() + else: + self.max_concurrency = max_concurrency if max_concurrency: self.queue = Queue() From 6384f09cab81f5af20bf13288d133f12406b0b74 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 6 Jul 2014 01:17:07 -0700 Subject: [PATCH 103/113] Don't block on kill incase worker is this process. --- gnsq/reader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gnsq/reader.py b/gnsq/reader.py index 47f363e..a94ef2a 100644 --- a/gnsq/reader.py +++ b/gnsq/reader.py @@ -262,14 +262,17 @@ def close(self): if not self.is_running: return + self.logger.debug('closing') self.state = CLOSED for worker in self.workers: - worker.kill() + worker.kill(block=False) for conn in self.conns: conn.close_stream() + self.logger.debug('workers: %r' % self.workers) + def join(self, timeout=None, raise_error=False): """Block until all connections have closed and workers stopped.""" gevent.joinall(self.workers, timeout, raise_error) From 89a8072f29d15d888694c22b03b1e2db08d2cd4d Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 6 Jul 2014 01:30:30 -0700 Subject: [PATCH 104/113] Fix tests to work in python 2.6 --- tests/integration_server.py | 10 ++++++++++ tests/test_lookupd.py | 9 +++++++-- tests/test_reader.py | 12 +++++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/integration_server.py b/tests/integration_server.py index 3bc55b5..32fa082 100644 --- a/tests/integration_server.py +++ b/tests/integration_server.py @@ -7,6 +7,16 @@ import urllib3 +def with_all(head, *tail): + def decorator(fn, *args): + with head as arg: + args = args + (arg,) + if not tail: + return fn(*args) + return with_all(*tail)(fn, *args) + return decorator + + class BaseIntegrationServer(object): http = urllib3.PoolManager() diff --git a/tests/test_lookupd.py b/tests/test_lookupd.py index ca53898..f4b3229 100644 --- a/tests/test_lookupd.py +++ b/tests/test_lookupd.py @@ -3,7 +3,11 @@ import pytest import gnsq -from integration_server import LookupdIntegrationServer, NsqdIntegrationServer +from integration_server import ( + with_all, + LookupdIntegrationServer, + NsqdIntegrationServer +) @pytest.mark.slow @@ -26,7 +30,8 @@ def test_lookup(): lookupd_server = LookupdIntegrationServer() nsqd_server = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) - with lookupd_server, nsqd_server: + @with_all(lookupd_server, nsqd_server) + def _(lookupd_server, nsqd_server): lookupd = gnsq.Lookupd(lookupd_server.http_address) conn = gnsq.Nsqd(nsqd_server.address, http_port=nsqd_server.http_port) diff --git a/tests/test_reader.py b/tests/test_reader.py index 11eeb75..ee4357e 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -7,7 +7,11 @@ from gnsq import Nsqd, Reader, states from gnsq.errors import NSQSocketError -from integration_server import NsqdIntegrationServer, LookupdIntegrationServer +from integration_server import ( + with_all, + LookupdIntegrationServer, + NsqdIntegrationServer +) def test_basic(): @@ -111,7 +115,8 @@ def test_max_concurrency(): server1 = NsqdIntegrationServer() server2 = NsqdIntegrationServer() - with server1, server2: + @with_all(server1, server2) + def _(server1, server2): class Accounting(object): count = 0 total = 100 @@ -176,7 +181,8 @@ def test_lookupd(): server1 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) server2 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) - with lookupd_server, server1, server2: + @with_all(lookupd_server, server1, server2) + def _(lookupd_server, server1, server2): class Accounting(object): count = 0 total = 500 From 4e8dde7ce68300e16bd67cf0e1290242270bbed8 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 6 Jul 2014 11:40:20 -0700 Subject: [PATCH 105/113] Better coverage build. --- Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index cc2baa4..2f16135 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,7 @@ help: @echo "release - package and upload a release" @echo "dist - package" -clean: clean-build clean-pyc clean-docs - rm -fr htmlcov/ +clean: clean-build clean-pyc clean-docs clean-coverage clean-tox clean-build: rm -fr build/ @@ -30,6 +29,13 @@ clean-docs: rm -f docs/modules.rst $(MAKE) -C docs clean +clean-coverage: + rm .coverage + rm -fr htmlcov/ + +clean-tox: + rm -fr .tox + lint: flake8 gnsq tests @@ -43,9 +49,7 @@ test-all: tox coverage: - py.test --runslow --cov gnsq tests - coverage report -m - coverage html + py.test --runslow --cov gnsq --cov-report html tests open htmlcov/index.html docs: clean-docs From ce157e79ed19a0f6d111570e5b6afe5f691a218a Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 6 Jul 2014 11:40:43 -0700 Subject: [PATCH 106/113] Update message test to expect signals. --- tests/test_message.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/test_message.py b/tests/test_message.py index b894658..466b4c8 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -7,21 +7,26 @@ def __init__(self, operations): self.operations = iter(operations) def __getattr__(self, name): - expected_name, expected_args = self.operations.next() - assert name == expected_name - def check_args(*args): + expected_name, expected_args = self.operations.next() + assert name == expected_name assert args == expected_args - return check_args def assert_finished(self): with pytest.raises(StopIteration): self.operations.next() + def connect_message(self, message): + message.on_finish.connect(self.finish) + message.on_requeue.connect(self.requeue) + message.on_touch.connect(self.touch) + def test_basic(): - message = gnsq.Message(None, 0, 42, '1234', 'sup') + message = gnsq.Message(0, 42, '1234', 'sup') + assert message.timestamp == 0 + assert message.attempts == 42 assert message.id == '1234' assert message.body == 'sup' assert message.has_responded() is False @@ -32,7 +37,8 @@ def test_finish(): ('finish', ('1234',)), ]) - message = gnsq.Message(mock_conn, 0, 42, '1234', 'sup') + message = gnsq.Message(0, 42, '1234', 'sup') + mock_conn.connect_message(message) assert message.has_responded() is False message.finish() @@ -47,7 +53,8 @@ def test_requeue(): ('requeue', ('1234', 0)), ]) - message = gnsq.Message(mock_conn, 0, 42, '1234', 'sup') + message = gnsq.Message(0, 42, '1234', 'sup') + mock_conn.connect_message(message) assert message.has_responded() is False message.requeue() @@ -64,7 +71,8 @@ def test_requeue_timeout(): ('requeue', ('1234', 1000)), ]) - message = gnsq.Message(mock_conn, 0, 42, '1234', 'sup') + message = gnsq.Message(0, 42, '1234', 'sup') + mock_conn.connect_message(message) assert message.has_responded() is False message.requeue(1000) @@ -84,7 +92,8 @@ def test_touch(): ('finish', ('1234',)), ]) - message = gnsq.Message(mock_conn, 0, 42, '1234', 'sup') + message = gnsq.Message(0, 42, '1234', 'sup') + mock_conn.connect_message(message) assert message.has_responded() is False message.touch() From 9c616f170102f66541e98c9e1184d0cd5e31ce8a Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 6 Jul 2014 12:07:45 -0700 Subject: [PATCH 107/113] Clean up mock connection. --- tests/test_message.py | 70 +++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/tests/test_message.py b/tests/test_message.py index 466b4c8..103ea39 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -3,25 +3,31 @@ class MockConnection(object): - def __init__(self, operations): + def __init__(self, message, operations): + message.on_finish.connect(self.finish) + message.on_requeue.connect(self.requeue) + message.on_touch.connect(self.touch) self.operations = iter(operations) - def __getattr__(self, name): - def check_args(*args): - expected_name, expected_args = self.operations.next() - assert name == expected_name - assert args == expected_args - return check_args + def finish(self, message): + exp_name, exp_args = self.operations.next() + assert exp_name == 'finish' + assert exp_args == (message,) + + def requeue(self, message, timeout): + exp_name, exp_args = self.operations.next() + assert exp_name == 'requeue' + assert exp_args == (message, timeout) + + def touch(self, message): + exp_name, exp_args = self.operations.next() + assert exp_name == 'touch' + assert exp_args == (message,) def assert_finished(self): with pytest.raises(StopIteration): self.operations.next() - def connect_message(self, message): - message.on_finish.connect(self.finish) - message.on_requeue.connect(self.requeue) - message.on_touch.connect(self.touch) - def test_basic(): message = gnsq.Message(0, 42, '1234', 'sup') @@ -33,12 +39,10 @@ def test_basic(): def test_finish(): - mock_conn = MockConnection([ - ('finish', ('1234',)), - ]) - message = gnsq.Message(0, 42, '1234', 'sup') - mock_conn.connect_message(message) + mock_conn = MockConnection(message, [ + ('finish', (message,)), + ]) assert message.has_responded() is False message.finish() @@ -47,14 +51,14 @@ def test_finish(): with pytest.raises(gnsq.errors.NSQException): message.finish() + mock_conn.assert_finished() -def test_requeue(): - mock_conn = MockConnection([ - ('requeue', ('1234', 0)), - ]) +def test_requeue(): message = gnsq.Message(0, 42, '1234', 'sup') - mock_conn.connect_message(message) + mock_conn = MockConnection(message, [ + ('requeue', (message, 0)), + ]) assert message.has_responded() is False message.requeue() @@ -67,12 +71,10 @@ def test_requeue(): def test_requeue_timeout(): - mock_conn = MockConnection([ - ('requeue', ('1234', 1000)), - ]) - message = gnsq.Message(0, 42, '1234', 'sup') - mock_conn.connect_message(message) + mock_conn = MockConnection(message, [ + ('requeue', (message, 1000)), + ]) assert message.has_responded() is False message.requeue(1000) @@ -85,15 +87,13 @@ def test_requeue_timeout(): def test_touch(): - mock_conn = MockConnection([ - ('touch', ('1234',)), - ('touch', ('1234',)), - ('touch', ('1234',)), - ('finish', ('1234',)), - ]) - message = gnsq.Message(0, 42, '1234', 'sup') - mock_conn.connect_message(message) + mock_conn = MockConnection(message, [ + ('touch', (message,)), + ('touch', (message,)), + ('touch', (message,)), + ('finish', (message,)), + ]) assert message.has_responded() is False message.touch() From 4e117eb7aa5f7352e744425fa852a289e9557df9 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Sun, 6 Jul 2014 12:18:14 -0700 Subject: [PATCH 108/113] Use signals for messages instead of keeping a reference to the connection. --- gnsq/message.py | 36 +++++++++++++++++++++++++++++------- gnsq/nsqd.py | 16 +++++++++++++++- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/gnsq/message.py b/gnsq/message.py index 1ac72ea..13eeb97 100644 --- a/gnsq/message.py +++ b/gnsq/message.py @@ -1,19 +1,41 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import blinker from .errors import NSQException class Message(object): - """A class representing a message received from nsqd.""" + """A class representing a message received from nsqd. - def __init__(self, conn, timestamp, attempts, id, body): - self._has_responded = False - self.conn = conn + **Signals:** + + .. data:: on_finish(reader, message) + :noindex: + + Sent when successfully finished. + + .. data:: on_requeue(reader, message, timeout) + :noindex: + + Sent when requeued. + + .. data:: on_touch(reader, message) + :noindex: + + Sent when touched. + """ + + def __init__(self, timestamp, attempts, id, body): self.timestamp = timestamp self.attempts = attempts self.id = id self.body = body + self._has_responded = False + self.on_finish = blinker.Signal() + self.on_requeue = blinker.Signal() + self.on_touch = blinker.Signal() + def has_responded(self): """Returns whether or not this message has been responded to.""" return self._has_responded @@ -26,7 +48,7 @@ def finish(self): if self._has_responded: raise NSQException('already responded') self._has_responded = True - self.conn.finish(self.id) + self.on_finish.send(self) def requeue(self, time_ms=0): """ @@ -36,10 +58,10 @@ def requeue(self, time_ms=0): if self._has_responded: raise NSQException('already responded') self._has_responded = True - self.conn.requeue(self.id, time_ms) + self.on_requeue.send(self, timeout=time_ms) def touch(self): """Respond to nsqd that you need more time to process the message.""" if self._has_responded: raise NSQException('already responded') - self.conn.touch(self.id) + self.on_touch.send(self) diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 63631e2..9256b32 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -243,10 +243,24 @@ def handle_message(self, data): self.last_message = time.time() self.ready_count -= 1 self.in_flight += 1 - message = Message(self, *nsq.unpack_message(data)) + + message = Message(*nsq.unpack_message(data)) + message.on_finish.connect(self.handle_finish) + message.on_requeue.connect(self.handle_requeue) + message.on_touch.connect(self.handle_touch) + self.on_message.send(self, message=message) return message + def handle_finish(self, message): + self.finish(message.id) + + def handle_requeue(self, message, timeout): + self.requeue(message.id, timeout) + + def handle_touch(self, message): + self.touch(message.id) + def finish_inflight(self): self.in_flight -= 1 From fc8a98733cfe1c0d3107a24199ce863ef0c7d791 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 7 Jul 2014 13:54:06 -0700 Subject: [PATCH 109/113] s/protocal/protocol/ --- gnsq/lookupd.py | 2 +- gnsq/nsqd.py | 2 +- gnsq/{protocal.py => protocol.py} | 0 tests/test_basic.py | 2 +- tests/test_command.py | 2 +- tests/test_nsqd.py | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename gnsq/{protocal.py => protocol.py} (100%) diff --git a/gnsq/lookupd.py b/gnsq/lookupd.py index 183612e..cd078ea 100644 --- a/gnsq/lookupd.py +++ b/gnsq/lookupd.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import from .httpclient import HTTPClient -from . import protocal as nsq +from . import protocol as nsq class Lookupd(HTTPClient): diff --git a/gnsq/nsqd.py b/gnsq/nsqd.py index 9256b32..c3e117e 100644 --- a/gnsq/nsqd.py +++ b/gnsq/nsqd.py @@ -10,7 +10,7 @@ except ImportError: import json # pyflakes.ignore -from . import protocal as nsq +from . import protocol as nsq from . import errors from .message import Message diff --git a/gnsq/protocal.py b/gnsq/protocol.py similarity index 100% rename from gnsq/protocal.py rename to gnsq/protocol.py diff --git a/tests/test_basic.py b/tests/test_basic.py index 0a4b9a6..2fcb0f9 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -1,7 +1,7 @@ import pytest from gnsq import BackoffTimer -from gnsq import protocal as nsq +from gnsq import protocol as nsq @pytest.mark.parametrize('name,good', [ diff --git a/tests/test_command.py b/tests/test_command.py index c9dbf37..ee31ae3 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -6,7 +6,7 @@ except ImportError: import json # pyflakes.ignore -from gnsq import protocal as nsq +from gnsq import protocol as nsq def pytest_generate_tests(metafunc): diff --git a/tests/test_nsqd.py b/tests/test_nsqd.py index 31d46d2..b11cbbc 100644 --- a/tests/test_nsqd.py +++ b/tests/test_nsqd.py @@ -5,7 +5,7 @@ import pytest from gnsq import Nsqd, Message, states, errors -from gnsq import protocal as nsq +from gnsq import protocol as nsq from gnsq.stream.stream import SSLSocket, DefalteSocket, SnappySocket from mock_server import mock_server From e45f8fc2788a2e65e30ee916b2be08e260cf756a Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 7 Jul 2014 13:54:35 -0700 Subject: [PATCH 110/113] Default test to run all tests. --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2f16135..3fbfd95 100644 --- a/Makefile +++ b/Makefile @@ -40,11 +40,11 @@ lint: flake8 gnsq tests test: - py.test - -test-slow: py.test --runslow +test-fast: + py.test + test-all: tox From 3078d0ebb52ff602af65436beeaec3f7680122ab Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 7 Jul 2014 14:23:41 -0700 Subject: [PATCH 111/113] Add clean phonys. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3fbfd95..f068f44 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean-pyc clean-build clean-docs docs clean +.PHONY: clean-pyc clean-build clean-pyc clean-docs clean-coverage clean-tox docs clean help: @echo "clean-build - remove build artifacts" From 3531d06779e76242649244d991f7c7f05a99bb10 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 7 Jul 2014 16:23:12 -0700 Subject: [PATCH 112/113] Don't die if .coverage doesn't exist. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f068f44..eaa6eba 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ clean-docs: $(MAKE) -C docs clean clean-coverage: - rm .coverage + rm -f .coverage rm -fr htmlcov/ clean-tox: From 104c7419a59c22fb75a99555d3b2f7920e48a777 Mon Sep 17 00:00:00 2001 From: William Trevor Olson Date: Mon, 7 Jul 2014 16:24:24 -0700 Subject: [PATCH 113/113] Set actual release date. --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index d1ee0b9..a072281 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ History ------- -0.1.0 (2014-01-11) +0.1.0 (2014-07-07) ~~~~~~~~~~~~~~~~~~ * First release on PyPI.