pax_global_header00006660000000000000000000000064136436324570014527gustar00rootroot0000000000000052 comment=f87e2da2758e22589adf482bfe10128868dc100f simplekv-0.14.1/000077500000000000000000000000001364363245700134445ustar00rootroot00000000000000simplekv-0.14.1/.gitignore000066400000000000000000000022331364363245700154340ustar00rootroot00000000000000# Created by .ignore support plugin (hsz.mobi) ### Python template # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # dotenv .env # virtualenv .venv venv/ ENV/ # Spyder project settings .spyderproject # Rope project settings .ropeproject # PyCharm .idea/ simplekv-0.14.1/.travis.yml000066400000000000000000000022661364363245700155630ustar00rootroot00000000000000language: python cache: pip sudo: false matrix: include: - python: 2.7 env: - TOXENV=py27 - python: 3.5 env: - TOXENV=py35 - python: 3.6 env: - TOXENV=py36 - python: 3.7 env: - TOXENV=py37 - python: 3.8 env: - TOXENV=py38 - python: 3.6 env: - TOXENV=py36-old-azure-storage-blob services: - docker - redis-server - mongodb - postgresql - mysql install: pip install tox coveralls before_script: - bash .travis/start_minio.sh - docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 & - psql -c 'create database simplekv_test;' -U postgres - psql -c 'ALTER ROLE travis CONNECTION LIMIT -1;' -U postgres - mysql -e 'create database simplekv_test;' - mysql -e 'set global max_connections = 200;' script: tox after_success: - coveralls deploy: provider: pypi user: "fxmc" password: secure: "bTmsf99BGyjIYj8NxMoWcKu2pdynHF0mUY6q3QgP7mnq430n0UONEJ/fdS9Q73mlouRzAjvdBESvwh9474g9RO778N3HTzn0XTLWpY50gRvblG8yD49x6g3+/IPPJ6ZifrViiiw7tbM4fZhQdPXd0t8vDrSZAVV6Z1v15NsWfkQ=" distributions: sdist bdist_wheel on: tags: true skip_existing: true simplekv-0.14.1/.travis/000077500000000000000000000000001364363245700150325ustar00rootroot00000000000000simplekv-0.14.1/.travis/start_minio.sh000066400000000000000000000003711364363245700177170ustar00rootroot00000000000000#!/usr/bin/env bash set -e set -x wget -O ~/minio https://dl.min.io/server/minio/release/linux-amd64/minio chmod +x ~/minio export MINIO_ACCESS_KEY=minio export MINIO_SECRET_KEY=miniostorage mkdir -p ~/s3 ~/minio --version ~/minio server ~/s3 & simplekv-0.14.1/AUTHORS000066400000000000000000000002441364363245700145140ustar00rootroot00000000000000Marc Brinkmann Cornelius Riemenschneider Felix Marczinowski simplekv-0.14.1/LICENSE000066400000000000000000000020421364363245700144470ustar00rootroot00000000000000Copyright (c) 2015 Marc Brinkmann Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. simplekv-0.14.1/MANIFEST.in000066400000000000000000000001251364363245700152000ustar00rootroot00000000000000include README.rst include LICENSE global-exclude *.swp exclude boto_credentials.ini simplekv-0.14.1/README.rst000066400000000000000000000021621364363245700151340ustar00rootroot00000000000000simple key-value storage api ============================ .. image:: https://travis-ci.org/mbr/simplekv.png?branch=master :target: https://travis-ci.org/mbr/simplekv .. image:: https://coveralls.io/repos/github/mbr/simplekv/badge.svg?branch=master :target: https://coveralls.io/github/mbr/simplekv?branch=master *simplekv* is an API for very basic key-value stores used for small, frequently accessed data or large binary blobs. Its basic interface is easy to implement and it supports a number of backends, including the filesystem, SQLAlchemy, MongoDB, Redis and Amazon S3/Google Storage. Installation ------------ simplekv is `available on PyPI `_ and can be installed through `pip `_:: pip install simplekv or via ``conda`` on `conda-forge `_:: conda install -c conda-forge simplekv Documentation ------------- The documentation for simplekv is available at https://simplekv.readthedocs.io. License ------- simplekv is `MIT licensed `_. simplekv-0.14.1/azure_credentials.ini000066400000000000000000000010411364363245700176440ustar00rootroot00000000000000# Specify the azure account to use in the tests. # The default assumes you are running azurite (https://github.com/Azure/Azurite). # To skip the azure store tests, remove the ``account_name``. # # Note that only the first config section is parsed. [azure-tests] account_name=devstoreaccount1 account_key=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== # these are not needed for a real azure blob store accounts; only set them for azurite: protocol=http endpoint=http://127.0.0.1:10000/devstoreaccount1 simplekv-0.14.1/boto_credentials.ini000066400000000000000000000002331364363245700174630ustar00rootroot00000000000000[minio-travis-test] access_key=minio secret_key=miniostorage connect_func=connect_s3 host=127.0.0.1 ordinary_calling_format=true is_secure=false port=9000 simplekv-0.14.1/docs/000077500000000000000000000000001364363245700143745ustar00rootroot00000000000000simplekv-0.14.1/docs/Makefile000066400000000000000000000060751364363245700160440ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = -n SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/simplekv.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/simplekv.qhc" latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." simplekv-0.14.1/docs/azure.rst000066400000000000000000000036561364363245700162660ustar00rootroot00000000000000Microsoft Azure Blob Storage **************************** Simplekv supports storing data in `Microsoft Azure Block Blob Storage `_. The backend uses the `azure-storage-blob `_ python distribution to access the azure blob storage and currently supports versions 2.x and 12.x. Note that ``azure-storage-blob`` is not a dependency for simplekv. You need to install it manually, otherwise you will see an :exc:`~exceptions.ImportError`. Here is a short example: :: from simplekv.net.azurestore import AzureBlockBlobStore conn_string = 'DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;' store = AzureBlockBlobStore(conn_string=conn_string, container='MyContainerName', public=False) # at this point, we can use the store like any other store.put(u'some-key', b'Hello, World!') # print out what's behind some-key. you should be able to see it # in the container now as well print store.get(u'some-key') Testing ======= The tests for the azure backend either * use a real azure blob store account or * use the `Azurite `_ blob storage emulator The travis tests use the second method. To test with a real blob store account, edit the file ``azure_credentials.ini`` s.t. the first config section contains the actual account_name and account_key of your test account. To test against a locally running azurite instance make sure to start azurite:: docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 & before running the tests. To skip the tests of the azure backend, comment out the ``account_name`` in the ``azure_credentials.ini`` file. .. autoclass:: simplekv.net.azurestore.AzureBlockBlobStore simplekv-0.14.1/docs/boto.rst000066400000000000000000000054351364363245700161000ustar00rootroot00000000000000.. cannot use auto-doc here, we do not want boto as a dependency for building the docs! Network and cloud-based storage ******************************* A core feature of simplekv is the ability to transparently store data using cloud storage services like `Amazon S3 `_ and `Google Storage `_. This is achieved by providing a backend that utilizes `boto `_ (preferably >= 2.25). Note that boto is not a dependency for simplekv. You need to install it manually, otherwise you will see an :exc:`~exceptions.ImportError`. Here is a short example: :: from simplekv.net.botostore import BotoStore import boto con = boto.connect_s3('your_access_key', 'your_secret_key') # use get_bucket instead, if you already have one! bucket = con.create_bucket('simplekv-testbucket') store = BotoStore(bucket) # at this point, we can use the store like any other store.put(u'some-key', b'Hello, World!') # print out what's behind some-key. you should be able to see it # in the bucket now as well print store.get(u'some-key') Unit testing ============ The unit-tests for the boto storage can only run if you have access to a Google Storage and/or Amazon S3 account. The tests will look in a file ``boto_credentials.ini`` in the simplekv source root folder for account credentials, here is an example file: :: [s3] access_key = YOUR_AMAZON_S3_ACCESS_KEY secret_key = YOUR_AMAZON_S3_SECRET_KEY connect_func = connect_s3 [gs] access_key = YOUR_GOOGLE_STORAGE_ACCESS_KEY secret_key = YOUR_GOOGLE_STORAGE_SECRET_KEY connect_func = connect_gs If a section is not present, the tests for that backend will be skipped. The unit tests for S3 will be run by travis against a local minio instance, emulating S3. .. class:: simplekv.net.boto.BotoStore Backend using the storage api of boto. .. method:: __init__(bucket, prefix='', url_valid_time=0, reduced_redundancy=False, public=False, metadata=None) Constructs a new boto based backend. :param bucket: An instance of :class:`boto.s3.bucket.Bucket`, :class:`boto.gs.bucket.Bucket` or similiar. :param prefix: A string that will transparently prefixed to all handled keys. :param url_valid_time=0: When using :meth:`~.UrlMixin.url_for`, URLs should be valid for this many seconds at most. :param reduced_redundancy: Use reduced redundancy storage for storing keys. :param public: If set, all newly updated values will be made public immediately. :param metadata: If set, for all newly created keys to be saved with these metadata values. simplekv-0.14.1/docs/cache.rst000066400000000000000000000017261364363245700161770ustar00rootroot00000000000000kv-caches ********* Caches speed up access to stores greatly, if used right. Usually, these require combining two :class:`~simplekv.KeyValueStore` instances of the same or different kind. A simple example without error-handling is a store that uses a :class:`~simplekv.memory.redisstore.RedisStore` in front of a :class:`~simplekv.fs.FilesystemStore`: :: from simplekv.memory.redisstore import RedisStore from simplekv.fs import FilesystemStore from simplekv.cache import CacheDecorator from redis import StrictRedis # initialize redis instance r = StrictRedis() store = CacheDecorator( cache=RedisStore(r), store=FilesystemStore('.') ) # will store the value in the FilesystemStore store.put(u'some_value', '123') # fetches from the FilesystemStore, but caches the result print store.get(u'some_value') # any further calls to store.get('some_value') will be served from the # RedisStore now .. automodule:: simplekv.cache :members: simplekv-0.14.1/docs/changes.rst000066400000000000000000000104761364363245700165460ustar00rootroot00000000000000Changelog ********* 0.14.1 ====== * Fix support for ``key in store`` for azure with ``azure-storage-blob`` 12 0.14.0 ====== * Add support for ``azure-storage-blob`` version 12. (``azure-storage-blob`` version 2 is still supported.) 0.13.1 ====== * Add the optional parameters of the Azure API max_block_size and max_single_put_size to the AzureBlockBlobStore. 0.13.0 ====== * Add ``iter_prefixes()`` method to iterate over all prefixes currently in the store, in any order. The prefixes are listed up to the given delimiter. 0.12.0 ====== * Use ``BlockBlobService.list_blob_names`` in :meth:`simplekv.net.azurestore.AzureBlockBlobStore.iter_keys``. This will only parse the names from Azure's XML response thus reducing CPU time siginificantly for this function. * They ``.keys()`` method on Python 3 now returns a list. This is in line with the documentation and the behaviour on Python 2. It used to return a generator. 0.11.11 ==== * Fix file-descriptor leak in `KeyValueStore._get_file` 0.11.10 ======= * Azure files handles now correctly implement seek and return the new position. 0.11.9 ====== * Add option to set the checksum for Azure blobs. * Make the FilesystemStore resilient to parallel directory creations. 0.11.8 ====== * Depend on azure-storage-blob, following the new naming scheme. * Pass the max_connections parameter to Azure backend. 0.11.7 ====== * removed seek() and tell() API for file handles opened in the botostore, due to it leaking HTTP connections to S3. 0.11.6 ====== * Support seek() and tell() API for file handles opened in the botostore. 0.11.5 ====== * Fix one off in open() method interfaces for azure backend 0.11.4 ====== * The open() method in the azure backend now supports partial reads of blobs * The exceptions from the azure backend contain more human-readable information in case of common errors. 0.11.3 ====== * Apply 0.11.2 in ExtendedKeySpaceMixin as well 0.11.2 ====== * Restore old behaviour of accepting valid keys of type `str` on Python 2 0.11.1 ====== * Fix version in setup.py 0.11.0 ====== * The memcached backend has been removed * Keys have to be provided as unicode strings * Values have to be provided as bytes (python 2) or as str (python 3) * keys() and iter_keys() provide a parameter to iterate just over all keys with a given prefix * Added :class:`simplekv.CopyMixin` to allow access to copy operations to backends which support a native copy operation * Added a decorator which provides a read-only view of a store: :class:`~simplekv.decorator.ReadOnlyDecorator` * Added a decorator which url-encodes all keys: :class:`~simplekv.decorator.URLEncodeKeysDecorator` * Added a Microsoft Azure Blob Storage backend: :class:`~simplekv.net.azurestore.AzureBlockBlobStore` * Added :class:`~simplekv.contrib.ExtendedKeyspaceMixin` which allows slashes and spaces in key names This mixin is experimental, unsupported and might not work with all backends. 0.10.0 ====== * simplekv no longer depends on ``six``. * The :class:`~simplekv.decorator.PrefixDecorator` works more as expected. * An experimental git-based store has been added in :class:`~simplekv.git.GitCommitStore`. 0.9.2 ===== * Added :class:`~simplekv.decorator.PrefixDecorator`. 0.9 === * Deprecated the :class:`~simplekv.UrlKeyValueStore`, replaced by flexible mixins like :class:`~simplekv.UrlMixin`. * Added :class:`~simplekv.TimeToLiveMixin` support (on :class:`~simplekv.memory.redisstore.RedisStore` and simplekv.memory.memcachestore.MemcacheStore). 0.6 === * Now supports `redis `_ backend: :class:`~simplekv.memory.redisstore.RedisStore`. * Fixed bug: No initial value for String() column in SQLAlchemy store. 0.5 === * Maximum key length that needs to be supported by all backends is 250 characters (was 256 before). * Added `memcached `_ backend: simplekv.memory.memcachestore.MemcacheStore * Added `SQLAlchemy `_ support: :class:`~simplekv.db.sql.SQLAlchemyStore` * Added :mod:`simplekv.cache` module. 0.4 === * Support for cloud-based storage using `boto `_ (see :class:`.BotoStore`). * First time changes were recorded in docs 0.3 === * **Major API Change**: Mixins replaced with decorators (see :class:`simplekv.idgen.HashDecorator` for an example) * Added `simplekv.crypt` 0.1 === * Initial release simplekv-0.14.1/docs/conf.py000066400000000000000000000151651364363245700157030ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # simplekv documentation build configuration file, created by # sphinx-quickstart on Tue May 10 09:07:13 2011. # # This file is execfile()d with the current directory set to its containing # dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os sys.path.append('../') on_rtd = os.environ.get('READTHEDOCS') == 'True' if on_rtd: html_theme = 'alabaster' else: import sphinx_readable_theme html_theme = 'readable' html_theme_path = [sphinx_readable_theme.get_html_theme_path()] # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.append(os.path.abspath('.')) # -- General configuration ---------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.intersphinx'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8' # The master toctree document. master_doc = 'index' # General information about the project. project = u'simplekv' copyright = u'2011, Marc Brinkmann' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '0.14.1' # The full version, including alpha/beta/rc tags. release = '0.14.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. #unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = ['_build'] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output ------------------------------------------------- # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". #html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_use_modindex = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = 'simplekvdoc' # -- Options for LaTeX output ------------------------------------------------ # The paper size ('letter' or 'a4'). #latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass # [howto/manual]). latex_documents = [ ('index', 'simplekv.tex', u'simplekv Documentation', u'Marc Brinkmann', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # Additional stuff for the LaTeX preamble. #latex_preamble = '' # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_use_modindex = True intersphinx_mapping = {'http://docs.python.org/': None, 'http://docs.pythonboto.org/en/latest/': None, 'http://sendapatch.se/projects/pylibmc/': None, 'http://www.sqlalchemy.org/docs/': None, 'http://redis-py.readthedocs.org/en/latest/': None} simplekv-0.14.1/docs/crypt.rst000066400000000000000000000001531364363245700162660ustar00rootroot00000000000000Hash and encryption decorators ****************************** .. automodule:: simplekv.crypt :members: simplekv-0.14.1/docs/db.rst000066400000000000000000000043401364363245700155140ustar00rootroot00000000000000.. cannot use auto-doc here, sqlalchemy dependency! Database-based stores ********************* SQLAlchemy ---------- To store data in existing databases, an `SQLAlchemy `_ based store is available: :: from simplekv.db.sql import SQLAlchemyStore from sqlalchemy import create_engine, MetaData # use echo=True to see queries engine = create_engine('sqlite:///:memory:', echo=True) metadata = MetaData(bind=engine) # init store store = SQLAlchemyStore(engine, metadata, 'kvstore') # create the actual table in the database (only do this once!) metadata.create_all() # also possible: store.table.create() # use store normally store.put(u'my_key', 'some value') print store.get(u'my_key') .. class:: simplekv.db.sql.SQLAlchemyStore Stores data in a table in a database through `SQLAlchemy `_. Note that this storage is not well-suited for large binary data, as currently it does not support streaming of large blobs. In other words, every value must be read into memory, before it can be returned. .. method:: __init__(bind, metadata, tablename) Generates a new :class:`~sqlalchemy.schema.Table` for use as a backend (see :attr:`~simplekv.db.sql.SQLAlchemyStore.table`) on the supplied metadata. :param bind: Any queries made by the store run :meth:`~sqlalchemy.sql.expression.Executable.execute` using this bind. :param metadata: :class:`sqlalchemy.schema.MetaData` instance on which the table will be created. :param tablename: The name for the table. .. attribute:: table An :class:`sqlalchemy.schema.Table` instance autogenerated by :meth:`__init__`. Calling :meth:`~sqlalchemy.schema.Table.create` can be used to create the table in the database. MongoDB ------- The :class:`~simplekv.db.mongo.MongoStore` class requires the ``pymongo`` package to be installed. .. class:: simplekv.db.mongo.MongoStore .. method:: __init__(db, collection) Uses a MongoDB collection as the backend, using pickle as a serializer. :param db: A (already authenticated) pymongo database. :param collection: A MongoDB collection name. simplekv-0.14.1/docs/decorators.rst000066400000000000000000000003021364363245700172660ustar00rootroot00000000000000Other decorators **************** .. autoclass:: simplekv.decorator.PrefixDecorator .. autoclass:: simplekv.decorator.URLEncodeKeysDecorator .. autoclass:: simplekv.decorator.ReadOnlyDecorator simplekv-0.14.1/docs/development.rst000066400000000000000000000004351364363245700174520ustar00rootroot00000000000000Development *********** Github ====== All official development of the library takes place on `GitHub `_. Comments, bug reports and patches are usually welcome, if you have any questions regarding the library, you can message the author there as well. simplekv-0.14.1/docs/filesystem.rst000066400000000000000000000005301364363245700173100ustar00rootroot00000000000000Filesystem-based key-value stores ********************************* A straightforward implementation is a filesystem-based implementation found in :class:`simplekv.fs.FilesystemStore` class, as well as a slightly altered version suitable for web applications, :class:`simplekv.fs.WebFilesystemStore`. .. automodule:: simplekv.fs :members: simplekv-0.14.1/docs/gae.rst000066400000000000000000000017641364363245700156720ustar00rootroot00000000000000Google app engine support ************************* Currently, there is support for storing data on `Google app engine `_ using ndb. This is mainly intended for storing small bits of data, like sessions or similiar but not large binary blobs [1]_. .. class:: simplekv.gae.NdbStore A backend that stores data on ndb objects. The objects key will be the key in the kvstore, while an attribute ``v`` must be present on the object to store the data. .. method:: __init__(obj_class): :param obj_class: An instance of a `Model `_ object. Here is a minimal example: :: from google.appengine.ext import ndb from simplekv.gae import NdbStore class MyKvModel(ndb.Model): v = ndb.BlobProperty(indexed=False) store = NdbStore(MyKvModel) .. [1] You're welcome to implement it and `submit a patch `_. simplekv-0.14.1/docs/git.rst000066400000000000000000000017501364363245700157140ustar00rootroot00000000000000Git-based stores **************** .. warning:: The git-based store here passes all tests, but may perform slower than expected. The code should be considered beta quality in this release. The functionality in this module requires the dulwich_ module (but no git binary). Repositories are manipulated directly, i.e. possibly existing working copies are not touched. .. _dulwich: http://dulwich.io .. class:: simplekv.git.GitCommitStore(repo_path, branch=b'master',\ subdir=b'') A git-commit based store. For every put or delete operation, a new commit will be created in a git repository. Keys themselves will map onto file-paths in the repository, possibly in a subdirectory. :param repo_path: Path to the git repository. :param branch: The branch to commit to. Must be an ascii-encoded binary string. :param subdir: Prefixed to every key committed. Must be an ascii-encoded binary string. simplekv-0.14.1/docs/idgen.rst000066400000000000000000000001371364363245700162150ustar00rootroot00000000000000ID-generating decorators ************************ .. automodule:: simplekv.idgen :members: simplekv-0.14.1/docs/index.rst000066400000000000000000000134621364363245700162430ustar00rootroot00000000000000simple key-value storage api **************************** *simplekv* is an API for key-value store of binary data. Due to its basic interface, it is easy to implemented a large number of backends. *simplekv*'s origins are in storing user-uploaded files on websites, but its low overhead and design should make it applicable for numerous other problems, an example is a `session backend for the Flask framework `_. Built upon the solid foundation are a few optional bells and whistles, such as automatic ID generation/hashing (in :mod:`simplekv.idgen`). A number of backends are available, ranging from :class:`~.FilesystemStore` to support for `Amazon S3 `_ and `Google Storage `_ through :class:`~.BotoStore`. A faster in-memory store suitable for session management and caching is supported through :class:`~.RedisStore` Example ======= Here's a simple example:: from simplekv.fs import FilesystemStore store = FilesystemStore('./data') store.put(u'key1', 'hello') # will print "hello" print store.get(u'key1') # move the contents of a file to "key2" as efficiently as possible store.put_file(u'key2', '/path/to/data') Note that by changing the first two lines to:: from simplekv.memory.redisstore import RedisStore import redis store = RedisStore(redis.StrictRedis()) you could use the code exactly the same way, this time storing data inside a Redis database. Why you should use simplekv ============================ no server dependencies *simplekv* does only depend on python and possibly a few libraries easily fetchable from PyPI_, if you want to use extra features. You do not have to run and install any server software to use *simplekv* (but can at any point later on). specializes in (even large!) blobs The fastest, most basic *simplekv* backend implementation stores files on your harddrive and is just as fast. This underlines the focus on storing big blobs without overhead or metadata. A typical usecase is starting out small with local files and then migrating all your binary data to something like Amazon's S3_. .. _PyPI: http://pypi.python.org .. _S3: https://s3.amazonaws.com/ Table of contents ================= .. toctree:: :maxdepth: 3 filesystem boto azure gae memory db git idgen crypt decorators cache development changes The core API ============ .. autoclass:: simplekv.KeyValueStore :members: __contains__, __iter__, delete, get, get_file, iter_keys, keys, open, put, put_file Some backends support an efficient copy operation, which is provided by a mixin class: .. autoclass:: simplekv.CopyMixin :members: copy In addition to that, a mixin class is available for backends that provide a method to support URL generation: .. autoclass:: simplekv.UrlMixin :members: url_for .. autoclass:: simplekv.UrlKeyValueStore :members: Some backends support setting a time-to-live on keys for automatic expiration, this is represented by the :class:`~simplekv.TimeToLiveMixin`: .. autoclass:: simplekv.TimeToLiveMixin .. automethod:: simplekv.TimeToLiveMixin.put .. automethod:: simplekv.TimeToLiveMixin.put_file .. attribute:: default_ttl_secs = simplekv.NOT_SET Passing ``None`` for any time-to-live parameter will cause this value to be used. .. autoattribute:: simplekv.TimeToLiveMixin.ttl_support .. autodata:: simplekv.VALID_KEY_REGEXP .. autodata:: simplekv.VALID_KEY_RE .. _implement: Implementing a new backend ========================== Subclassing :class:`~simplekv.KeyValueStore` is the fastest way to implement a new backend. It suffices to override the :func:`~simplekv.KeyValueStore._delete`, :func:`~simplekv.KeyValueStore.iter_keys`, :func:`~simplekv.KeyValueStore._open` and :func:`~simplekv.KeyValueStore._put_file` methods, as all the other methods have default implementations that call these. After that, you can override any number of underscore-prefixed methods with more specialized implementations to gain speed improvements. Default implementation ---------------------- Classes derived from :class:`~simplekv.KeyValueStore` inherit a number of default implementations for the core API methods. Specifically, the :func:`~simplekv.KeyValueStore.delete`, :func:`~simplekv.KeyValueStore.get`, :func:`~simplekv.KeyValueStore.get_file`, :func:`~simplekv.KeyValueStore.keys`, :func:`~simplekv.KeyValueStore.open`, :func:`~simplekv.KeyValueStore.put`, :func:`~simplekv.KeyValueStore.put_file`, methods will each call the :func:`~simplekv.KeyValueStore._check_valid_key` method if a key has been provided and then call one of the following protected methods: .. automethod:: simplekv.KeyValueStore._check_valid_key .. automethod:: simplekv.KeyValueStore._delete .. automethod:: simplekv.KeyValueStore._get .. automethod:: simplekv.KeyValueStore._get_file .. automethod:: simplekv.KeyValueStore._get_filename .. automethod:: simplekv.KeyValueStore._has_key .. automethod:: simplekv.KeyValueStore._open .. automethod:: simplekv.KeyValueStore._put .. automethod:: simplekv.KeyValueStore._put_file .. automethod:: simplekv.KeyValueStore._put_filename Atomicity ========= Every call to a method on a KeyValueStore results in a single operation on the underlying backend. No guarantees are made above that, if you check if a key exists and then try to retrieve it, it may have already been deleted in between (instead, retrieve and catch the exception). Python 3 ======== All of the examples are written in Python 2. However, Python 3 is fully supported and tested. When using *simplekv* in a Python 3 environment, the only important thing to remember is that keys are always strings and values are always byte-objects. Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` simplekv-0.14.1/docs/make.bat000066400000000000000000000060031364363245700160000ustar00rootroot00000000000000@ECHO OFF REM Command file for Sphinx documentation set SPHINXBUILD=sphinx-build set BUILDDIR=_build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% ) if "%1" == "" goto help if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files echo. dirhtml to make HTML files named index.html in directories echo. pickle to make pickle files echo. json to make JSON files echo. htmlhelp to make HTML files and a HTML help project echo. qthelp to make HTML files and a qthelp project echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. changes to make an overview over all changed/added/deprecated items echo. linkcheck to check all external links for integrity echo. doctest to run all doctests embedded in the documentation if enabled goto end ) if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* goto end ) if "%1" == "html" ( %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html. goto end ) if "%1" == "dirhtml" ( %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml echo. echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. goto end ) if "%1" == "pickle" ( %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle echo. echo.Build finished; now you can process the pickle files. goto end ) if "%1" == "json" ( %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json echo. echo.Build finished; now you can process the JSON files. goto end ) if "%1" == "htmlhelp" ( %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp echo. echo.Build finished; now you can run HTML Help Workshop with the ^ .hhp project file in %BUILDDIR%/htmlhelp. goto end ) if "%1" == "qthelp" ( %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: echo.^> qcollectiongenerator %BUILDDIR%\qthelp\simplekv.qhcp echo.To view the help file: echo.^> assistant -collectionFile %BUILDDIR%\qthelp\simplekv.ghc goto end ) if "%1" == "latex" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex echo. echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. goto end ) if "%1" == "changes" ( %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes echo. echo.The overview file is in %BUILDDIR%/changes. goto end ) if "%1" == "linkcheck" ( %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck echo. echo.Link check complete; look for any errors in the above output ^ or in %BUILDDIR%/linkcheck/output.txt. goto end ) if "%1" == "doctest" ( %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest echo. echo.Testing of doctests in the sources finished, look at the ^ results in %BUILDDIR%/doctest/output.txt. goto end ) :end simplekv-0.14.1/docs/memory.rst000066400000000000000000000014001364363245700164310ustar00rootroot00000000000000In-memory stores **************** The simplest conceivable key-value store is found in :class:`simplekv.memory.DictStore`. It lends itself well for testing or playing around with *simplekv*. .. autoclass:: simplekv.memory.DictStore :members: redis-backend ============= The redis_-backend requires :py:mod:`redis` to be installed and uses a redis_-database as a backend. Currently, simplekv targets the latest stable redis version, but since it uses very few commands available, it should work on older stables as well. Note that some features are unsupported on older redis_-versions (such as sub-second accuracy for TTL values on redis_ < 2.6) and will cause redis to complain. .. autoclass:: simplekv.memory.redisstore.RedisStore .. _redis: http://redis.io simplekv-0.14.1/setup.py000066400000000000000000000013201364363245700151520ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 import os from setuptools import setup, find_packages def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() setup(name='simplekv', version='0.14.1', description=('A key-value storage for binary data, support many ' 'backends.'), long_description=read('README.rst'), author='Marc Brinkmann', author_email='git@marcbrinkmann.de', url='http://github.com/mbr/simplekv', license='MIT', packages=find_packages(exclude=['test']), install_requires=[], classifiers=[ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', ]) simplekv-0.14.1/simplekv/000077500000000000000000000000001364363245700152765ustar00rootroot00000000000000simplekv-0.14.1/simplekv/__init__.py000066400000000000000000000405451364363245700174170ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 import re from io import BytesIO from ._compat import key_type __version__ = '0.14.1' VALID_NON_NUM = r"""\`\!"#$%&'()+,-.<=>?@[]^_{}~""" VALID_KEY_REGEXP = "^[%s0-9a-zA-Z]+$" % re.escape(VALID_NON_NUM) """This regular expression tests if a key is valid. Allowed are all alphanumeric characters, as well as ``!"`#$%&'()+,-.<=>?@[]^_{}~``.""" VALID_KEY_RE = re.compile(VALID_KEY_REGEXP) """A compiled version of :data:`~simplekv.VALID_KEY_REGEXP`.""" class KeyValueStore(object): """The smallest API supported by all backends. Keys are ascii-strings with certain restrictions, guaranteed to be properly handled up to a length of at least 250 characters. Any function that takes a key as an argument raises a ValueError if the key is incorrect. The regular expression for what constitutes a valid key is available as :data:`simplekv.VALID_KEY_REGEXP`. Values are raw bytes. If you need to store strings, make sure to encode them upon storage and decode them upon retrieval. """ def __contains__(self, key): """Checks if a key is present :param key: The key whose existence should be verified. :raises exceptions.ValueError: If the key is not valid. :raises exceptions.IOError: If there was an error accessing the store. :returns: True if the key exists, False otherwise. """ self._check_valid_key(key) return self._has_key(key) def __iter__(self): """Iterate over keys :raises exceptions.IOError: If there was an error accessing the store. """ return self.iter_keys() def delete(self, key): """Delete key and data associated with it. If the key does not exist, no error is reported. :raises exceptions.ValueError: If the key is not valid. :raises exceptions.IOError: If there was an error deleting. """ self._check_valid_key(key) return self._delete(key) def get(self, key): """Returns the key data as a bytestring. :param key: Value associated with the key, as a `bytes` object :raises exceptions.ValueError: If the key is not valid. :raises exceptions.IOError: If the file could not be read. :raises exceptions.KeyError: If the key was not found. """ self._check_valid_key(key) return self._get(key) def get_file(self, key, file): """Write contents of key to file Like :meth:`.KeyValueStore.put_file`, this method allows backends to implement a specialized function if data needs to be written to disk or streamed. If *file* is a string, contents of *key* are written to a newly created file with the filename *file*. Otherwise, the data will be written using the *write* method of *file*. :param key: The key to be read :param file: Output filename or an object with a *write* method. :raises exceptions.ValueError: If the key is not valid. :raises exceptions.IOError: If there was a problem reading or writing data. :raises exceptions.KeyError: If the key was not found. """ self._check_valid_key(key) if isinstance(file, str): return self._get_filename(key, file) else: return self._get_file(key, file) def iter_keys(self, prefix=u""): """Return an Iterator over all keys currently in the store, in any order. If prefix is not the empty string, iterates only over all keys starting with prefix. :raises exceptions.IOError: If there was an error accessing the store. """ raise NotImplementedError def iter_prefixes(self, delimiter, prefix=u""): """Returns an Iterator over all prefixes currently in the store, in any order. The prefixes are listed up to the given delimiter. If the prefix contains the delimiter, the first delimiter after the prefix is used as a cut-off point. The uniqueness of the prefixes is ensured. The default uses an naive key iteration. Some backends may implement more efficient variants. :raises exceptions.IOError: If there was an error accessing the store. """ dlen = len(delimiter) plen = len(prefix) memory = set() for k in self.iter_keys(prefix): pos = k.find(delimiter, plen) if pos >= 0: k = k[: pos + dlen] if k not in memory: yield k memory.add(k) def keys(self, prefix=u""): """Return a list of keys currently in store, in any order If prefix is not the empty string, returns only all keys starting with prefix. :raises exceptions.IOError: If there was an error accessing the store. """ return list(self.iter_keys(prefix)) def open(self, key): """Open key for reading. Returns a read-only file-like object for reading a key. :param key: Key to open :raises exceptions.ValueError: If the key is not valid. :raises exceptions.IOError: If the file could not be read. :raises exceptions.KeyError: If the key was not found. """ self._check_valid_key(key) return self._open(key) def put(self, key, data): """Store into key from file Stores bytestring *data* in *key*. :param key: The key under which the data is to be stored :param data: Data to be stored into key, must be `bytes`. :returns: The key under which data was stored :raises exceptions.ValueError: If the key is not valid. :raises exceptions.IOError: If storing failed or the file could not be read """ self._check_valid_key(key) if not isinstance(data, bytes): raise IOError("Provided data is not of type bytes") return self._put(key, data) def put_file(self, key, file): """Store into key from file on disk Stores data from a source into key. *file* can either be a string, which will be interpretet as a filename, or an object with a *read()* method. If the passed object has a *fileno()* method, it may be used to speed up the operation. The file specified by *file*, if it is a filename, may be removed in the process, to avoid copying if possible. If you need to make a copy, pass the opened file instead. :param key: The key under which the data is to be stored :param file: A filename or an object with a read method. If a filename, may be removed :returns: The key under which data was stored :raises exceptions.ValueError: If the key is not valid. :raises exceptions.IOError: If there was a problem moving the file in. """ # FIXME: shouldn't we call self._check_valid_key here? if isinstance(file, str): return self._put_filename(key, file) else: return self._put_file(key, file) def _check_valid_key(self, key): """Checks if a key is valid and raises a ValueError if its not. When in need of checking a key for validity, always use this method if possible. :param key: The key to be checked """ if not isinstance(key, key_type): raise ValueError('%r is not a valid key type' % key) if not VALID_KEY_RE.match(key): raise ValueError('%r contains illegal characters' % key) def _delete(self, key): """Implementation for :meth:`~simplekv.KeyValueStore.delete`. The default implementation will simply raise a :py:exc:`~exceptions.NotImplementedError`. """ raise NotImplementedError def _get(self, key): """Implementation for :meth:`~simplekv.KeyValueStore.get`. The default implementation will create a :class:`io.BytesIO`-buffer and then call :meth:`~simplekv.KeyValueStore._get_file`. :param key: Key of value to be retrieved """ buf = BytesIO() self._get_file(key, buf) return buf.getvalue() def _get_file(self, key, file): """Write key to file-like object file. Either this method or :meth:`~simplekv.KeyValueStore._get_filename` will be called by :meth:`~simplekv.KeyValueStore.get_file`. Note that this method does not accept strings. :param key: Key to be retrieved :param file: File-like object to write to """ bufsize = 1024 * 1024 # note: we do not use a context manager here or close the source. # the source goes out of scope shortly after, taking care of the issue # this allows us to support file-like objects without close as well, # such as BytesIO. source = self.open(key) try: while True: buf = source.read(bufsize) file.write(buf) if len(buf) < bufsize: break finally: source.close() def _get_filename(self, key, filename): """Write key to file. Either this method or :meth:`~simplekv.KeyValueStore._get_file` will be called by :meth:`~simplekv.KeyValueStore.get_file`. This method only accepts filenames and will open the file with a mode of ``wb``, then call :meth:`~simplekv.KeyValueStore._get_file`. :param key: Key to be retrieved :param filename: Filename to write to """ with open(filename, 'wb') as dest: return self._get_file(key, dest) def _has_key(self, key): """Default implementation for :meth:`~simplekv.KeyValueStore.__contains__`. Determines whether or not a key exists by calling :meth:`~simplekv.KeyValueStore.keys`. :param key: Key to check existance of """ return key in self.keys() def _open(self, key): """Open key for reading. Default implementation simply raises a :py:exc:`~exceptions.NotImplementedError`. :param key: Key to open """ raise NotImplementedError def _put(self, key, data): """Implementation for :meth:`~simplekv.KeyValueStore.put`. The default implementation will create a :class:`io.BytesIO`-buffer and then call :meth:`~simplekv.KeyValueStore._put_file`. :param key: Key under which data should be stored :param data: Data to be stored """ return self._put_file(key, BytesIO(data)) def _put_file(self, key, file): """Store data from file-like object in key. Either this method or :meth:`~simplekv.KeyValueStore._put_filename` will be called by :meth:`~simplekv.KeyValueStore.put_file`. Note that this method does not accept strings. The default implementation will simply raise a :py:exc:`~exceptions.NotImplementedError`. :param key: Key under which data should be stored :param file: File-like object to store data from """ raise NotImplementedError def _put_filename(self, key, filename): """Store data from file in key. Either this method or :meth:`~simplekv.KeyValueStore._put_file` will be called by :meth:`~simplekv.KeyValueStore.put_file`. Note that this method does not accept strings. The default implementation will open the file in ``rb`` mode, then call :meth:`~simplekv.KeyValueStore._put_file`. :param key: Key under which data should be stored :param file: Filename of file to store """ with open(filename, 'rb') as source: return self._put_file(key, source) class UrlMixin(object): """Supports getting a download URL for keys.""" def url_for(self, key): """Returns a full external URL that can be used to retrieve *key*. Does not perform any checks (such as if a key exists), other than whether or not *key* is a valid key. :param key: The key for which the url is to be generated :raises exceptions.ValueError: If the key is not valid. :return: A string containing a URL to access key """ self._check_valid_key(key) return self._url_for(key) def _url_for(self, key): raise NotImplementedError FOREVER = 'forever' NOT_SET = 'not_set' class TimeToLiveMixin(object): """Allows keys to expire after a certain amount of time. This mixin overrides some of the signatures of the api of :class:`~simplekv.KeyValueStore`, albeit in a backwards compatible way. Any value given for a time-to-live parameter must be one of the following: * A positive ``int``, representing seconds, * ``simplekv.FOREVER``, meaning no expiration * ``simplekv.NOT_SET``, meaning that no TTL configuration will be done at all or * ``None`` representing the default (see :class:`.TimeToLiveMixin`'s ``default_ttl_secs``). .. note:: When deriving from :class:`~simplekv.TimeToLiveMixin`, the same default implementations for ``_put``, ``_put_file`` and ``_put_filename`` are provided, except that they all take an additional ``ttl_secs`` argument. For more information on how to implement backends, see :ref:`implement`. """ ttl_support = True """Indicates that a key-value store supports time-to-live features. This allows users of stores to test for support using:: getattr(store, 'ttl_support', False) """ default_ttl_secs = NOT_SET def _valid_ttl(self, ttl_secs): if ttl_secs is None: ttl_secs = self.default_ttl_secs if ttl_secs in (FOREVER, NOT_SET): return ttl_secs if not isinstance(ttl_secs, (int, float)): raise ValueError('Not a valid ttl_secs value: %r' % ttl_secs) if ttl_secs < 0: raise ValueError('ttl_secs must not be negative: %r' % ttl_secs) return ttl_secs def put(self, key, data, ttl_secs=None): """Like :meth:`~simplekv.KeyValueStore.put`, but with an additional parameter: :param ttl_secs: Number of seconds until the key expires. See above for valid values. :raises exceptions.ValueError: If ``ttl_secs`` is invalid. :raises exceptions.IOError: If storing failed or the file could not be read """ self._check_valid_key(key) if not isinstance(data, bytes): raise IOError("Provided data is not of type bytes") return self._put(key, data, self._valid_ttl(ttl_secs)) def put_file(self, key, file, ttl_secs=None): """Like :meth:`~simplekv.KeyValueStore.put_file`, but with an additional parameter: :param ttl_secs: Number of seconds until the key expires. See above for valid values. :raises exceptions.ValueError: If ``ttl_secs`` is invalid. """ if ttl_secs is None: ttl_secs = self.default_ttl_secs self._check_valid_key(key) if isinstance(file, str): return self._put_filename(key, file, self._valid_ttl(ttl_secs)) else: return self._put_file(key, file, self._valid_ttl(ttl_secs)) # default implementations similar to KeyValueStore below: def _put(self, key, data, ttl_secs): return self._put_file(key, BytesIO(data), ttl_secs) def _put_file(self, key, file, ttl_secs): raise NotImplementedError def _put_filename(self, key, filename, ttl_secs): with open(filename, 'rb') as source: return self._put_file(key, source, ttl_secs) class UrlKeyValueStore(UrlMixin, KeyValueStore): """ .. deprecated:: 0.9 Use the :class:`.UrlMixin` instead. """ pass class CopyMixin(object): """Exposes a copy operation, if the backend supports it.""" def copy(self, source, dest): """Copies a key. The destination is overwritten if does exist. :param source: The source key to copy :param dest: The destination for the copy :returns: The destination key :raises: exceptions.ValueError: If the source or target key are not valid :raises: exceptions.KeyError: If the source key was not found""" self._check_valid_key(source) self._check_valid_key(dest) return self._copy(source, dest) simplekv-0.14.1/simplekv/_compat.py000066400000000000000000000021541364363245700172740ustar00rootroot00000000000000"""Helpers for python 2/3 compatibility""" import sys PY2 = sys.version_info[0] == 2 if not PY2: from configparser import ConfigParser else: from ConfigParser import ConfigParser if not PY2: from urllib.parse import quote as url_quote, unquote as url_unquote from urllib.parse import quote_plus, unquote_plus else: from urllib import quote as url_quote from urllib import unquote as url_unquote from urllib import quote_plus, unquote_plus if not PY2: from urllib.parse import urlparse else: from urlparse import urlparse if not PY2: imap = map ifilter = filter else: from itertools import imap from itertools import ifilter if not PY2: from io import BytesIO else: from cStringIO import StringIO as BytesIO if not PY2: import pickle else: try: import cPickle as pickle except ImportError: import pickle xrange = range if not PY2 else xrange if not PY2: text_type = str key_type = str unichr = chr binary_type = bytes else: text_type = unicode key_type = basestring unichr = unichr binary_type = str simplekv-0.14.1/simplekv/cache.py000066400000000000000000000123471364363245700167220ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 from .decorator import StoreDecorator class CacheDecorator(StoreDecorator): """Write-through cache decorator. Can combine two :class:`~simplekv.KeyValueStore` instances into a single caching :class:`~simplekv.KeyValueStore`. On a data-read request, the cache will be consulted first, if there is a cache miss or an error, data will be read from the backing store. Any retrieved value will be stored in the cache before being forward to the client. Write, key-iteration and delete requests will be passed on straight to the backing store. After their completion, the cache will be updated. No cache maintainenace above this is done by the decorator. The caching store itselfs decides how large to grow the cache and which data to keep, which data to throw away. :param cache: The caching backend. :param store: The backing store. This is the "authorative" backend. """ def __init__(self, cache, store): super(CacheDecorator, self).__init__(store) self.cache = cache def delete(self, key): """Implementation of :meth:`~simplekv.KeyValueStore.delete`. If an exception occurs in either the cache or backing store, all are passing on. """ self._dstore.delete(key) self.cache.delete(key) def get(self, key): """Implementation of :meth:`~simplekv.KeyValueStore.get`. If a cache miss occurs, the value is retrieved, stored in the cache and returned. If the cache raises an :exc:`~exceptions.IOError`, the cache is ignored, and the backing store is consulted directly. It is possible for a caching error to occur while attempting to store the value in the cache. It will not be handled as well. """ try: return self.cache.get(key) except KeyError: # cache miss or error, retrieve from backend data = self._dstore.get(key) # store in cache and return self.cache.put(key, data) return data except IOError: # cache error, ignore completely and return from backend return self._dstore.get(key) def get_file(self, key, file): """Implementation of :meth:`~simplekv.KeyValueStore.get_file`. If a cache miss occurs, the value is retrieved, stored in the cache and returned. If the cache raises an :exc:`~exceptions.IOError`, the retrieval cannot proceed: If ``file`` was an open file, data maybe been written to it already. The :exc:`~exceptions.IOError` bubbles up. It is possible for a caching error to occur while attempting to store the value in the cache. It will not be handled as well. """ try: return self.cache.get_file(key, file) except KeyError: # cache miss, load into cache fp = self._dstore.open(key) self.cache.put_file(key, fp) # return from cache return self.cache.get_file(key, file) # if an IOError occured, file pointer may be dirty - cannot proceed # safely def open(self, key): """Implementation of :meth:`~simplekv.KeyValueStore.open`. If a cache miss occurs, the value is retrieved, stored in the cache, then then another open is issued on the cache. If the cache raises an :exc:`~exceptions.IOError`, the cache is ignored, and the backing store is consulted directly. It is possible for a caching error to occur while attempting to store the value in the cache. It will not be handled as well. """ try: return self.cache.open(key) except KeyError: # cache miss, load into cache fp = self._dstore.open(key) self.cache.put_file(key, fp) return self.cache.open(key) except IOError: # cache error, ignore completely and return from backend return self._dstore.open(key) def copy(self, source, dest): """Implementation of :meth:`~simplekv.CopyMixin.copy`. Copies the data in the backing store and removes the destination key from the cache, in case it was already populated. Does not work when the backing store does not implement copy. """ try: k = self._dstore.copy(source, dest) finally: self.cache.delete(dest) return k def put(self, key, data): """Implementation of :meth:`~simplekv.KeyValueStore.put`. Will store the value in the backing store. After a successful or unsuccessful store, the cache will be invalidated by deleting the key from it. """ try: return self._dstore.put(key, data) finally: self.cache.delete(key) def put_file(self, key, file): """Implementation of :meth:`~simplekv.KeyValueStore.put_file`. Will store the value in the backing store. After a successful or unsuccessful store, the cache will be invalidated by deleting the key from it. """ try: return self._dstore.put_file(key, file) finally: self.cache.delete(key) simplekv-0.14.1/simplekv/contrib/000077500000000000000000000000001364363245700167365ustar00rootroot00000000000000simplekv-0.14.1/simplekv/contrib/__init__.py000066400000000000000000000026721364363245700210560ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 import re from simplekv import VALID_NON_NUM from simplekv._compat import key_type VALID_NON_NUM_EXTENDED = VALID_NON_NUM + r"/ " VALID_KEY_REGEXP_EXTENDED = "^[%s0-9a-zA-Z]+$" % re.escape(VALID_NON_NUM_EXTENDED) """This regular expression tests if a key is valid when the extended keyspace mixin is used. Allowed are all alphanumeric characters, as well as ``!"`#$%&'()+,-.<=>?@[]^_{}~/``. and spaces""" VALID_KEY_RE_EXTENDED = re.compile(VALID_KEY_REGEXP_EXTENDED) """A compiled version of :data:`~simplekv.VALID_KEY_REGEXP_EXTENDED`.""" class ExtendedKeyspaceMixin(object): """A mixin to extend the keyspace to allow slashes and spaces in keynames. Attention: A single / is NOT allowed. Use it by extending first from ` :class:`~simplekv.ExtendedKeyspaceMixin` and then by the desired store. Note: This Mixin is unsupported and might not work correctly with all backends. """ def _check_valid_key(self, key): """Checks if a key is valid and raises a ValueError if its not. When in need of checking a key for validity, always use this method if possible. :param key: The key to be checked """ if not isinstance(key, key_type) and key is not None: raise ValueError('%r is not a valid key type' % key) if not VALID_KEY_RE_EXTENDED.match(key) or key == u'/': raise ValueError('%r contains illegal characters' % key) simplekv-0.14.1/simplekv/crypt.py000066400000000000000000000143611364363245700170160ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 import hashlib import hmac import os import tempfile from .decorator import StoreDecorator class _HMACFileReader(object): def __init__(self, hm, source): self.hm = hm self.source = source # "preload" buffer self.buffer = source.read(self.hm.digest_size) if not len(self.buffer) == self.hm.digest_size: raise VerificationException('Source does not contain HMAC hash ' '(too small)') def read(self, n=None): if b'' == self.buffer or 0 == n: return b'' new_read = self.source.read(n) if n is not None else self.source.read() finished = (n is None or len(new_read) != n) self.buffer += new_read if n is not None: offset = min(n, len(self.buffer) - self.hm.digest_size) else: offset = len(self.buffer) - self.hm.digest_size rv, self.buffer = self.buffer[:offset], self.buffer[offset:] # update hmac self.hm.update(rv) if finished: # check hash if not self.buffer == self.hm.digest(): raise VerificationException('HMAC verification failed.') return rv def close(self): self.source.close() def __enter__(self): return self def __exit__(self, *args): self.close() class VerificationException(Exception): """This exception is thrown whenever there was an error with an authenticity check performed by any of the decorators in this module.""" pass class HMACDecorator(StoreDecorator): """HMAC authentication and integrity check decorator. This decorator overrides the :meth:`.KeyValueStore.get`, :meth:`.KeyValueStore.get_file`, :meth:`.KeyValueStore.open`, :meth:`.KeyValueStore.put` and :meth:`.KeyValueStore.put_file` methods and alters the data that is store in the follow way: First, the original data is stored while being fed to an hmac instance. The resulting hash is appended to the data as a binary string, every value stored therefore takes up an additional ``hmac_digestsize`` bytes. Upon retrieval using any of :meth:`.KeyValueStore.get`, :meth:`.KeyValueStore.get_file` or :meth:`.KeyValueStore.open` methods, the data is checked as soon as the hash is readable. Since hashes are stored at the end, almost no extra memory is used when using streaming methods. However, :meth:`.KeyValueStore.get_file` and :meth:`.KeyValueStore.open` will only check the hash value once it is read, that is, at the end of the retrieval. The decorator will protect against any modification of the stored data and ensures that only those with knowledge of the ``__secret_key`` can alter any data. The key used to store data is also used to extend the HMAC secret key, making it impossible to copy a valid message over to a different key. """ def __init__(self, secret_key, decorated_store, hashfunc=hashlib.sha256): super(HMACDecorator, self).__init__(decorated_store) self.__hashfunc = hashfunc self.__secret_key = bytes(secret_key) def __new_hmac(self, key, msg=None): if not msg: msg = b'' # item key is used as salt for secret_key hm = hmac.HMAC( key=key.encode('ascii') + self.__secret_key, msg=msg, digestmod=self.__hashfunc) return hm def get(self, key): buf = self._dstore.get(key) hm = self.__new_hmac(key) hash = buf[-hm.digest_size:] # shorten buf buf = buf[:-hm.digest_size] hm.update(buf) if not hm.digest() == hash: raise VerificationException('Invalid hash on key %r' % key) return buf def get_file(self, key, file): if isinstance(file, str): try: f = open(file, 'wb') except (OSError, IOError) as e: raise IOError('Error opening %s for writing: %r' % ( file, e )) # file is open, now we call ourself again with a proper file try: self.get_file(key, f) finally: f.close() else: # need to use open, no way around it it seems # this will check the HMAC as well source = self.open(key) bufsize = 1024 * 1024 # copy while True: buf = source.read(bufsize) file.write(buf) if len(buf) != bufsize: break def open(self, key): source = self._dstore.open(key) return _HMACFileReader(self.__new_hmac(key), source) def put(self, key, value, *args, **kwargs): # just append hmac and put data = value + self.__new_hmac(key, value).digest() return self._dstore.put(key, data, *args, **kwargs) def copy(self, source, dest): raise NotImplementedError def put_file(self, key, file, *args, **kwargs): hm = self.__new_hmac(key) bufsize = 1024 * 1024 if isinstance(file, str): # we read the file once, then write the hash at the end, before # handing it over to the original backend with open(file, 'rb+') as source: while True: buf = source.read(bufsize) hm.update(buf) if len(buf) < bufsize: break # file has been read, append hash source.write(hm.digest()) # after the file has been closed, hand it over return self._dstore.put_file(key, file, *args, **kwargs) else: tmpfile = tempfile.NamedTemporaryFile(delete=False) try: while True: buf = file.read(bufsize) hm.update(buf) tmpfile.write(buf) if len(buf) < bufsize: break tmpfile.write(hm.digest()) tmpfile.close() return self._dstore.put_file( key, tmpfile.name, *args, **kwargs ) finally: os.unlink(tmpfile.name) simplekv-0.14.1/simplekv/db/000077500000000000000000000000001364363245700156635ustar00rootroot00000000000000simplekv-0.14.1/simplekv/db/__init__.py000066400000000000000000000000001364363245700177620ustar00rootroot00000000000000simplekv-0.14.1/simplekv/db/mongo.py000066400000000000000000000026341364363245700173610ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8 -*- from .. import KeyValueStore from .._compat import BytesIO from .._compat import pickle from bson.binary import Binary import re class MongoStore(KeyValueStore): """Uses a MongoDB collection as the backend, using pickle as a serializer. :param db: A (already authenticated) pymongo database. :param collection: A MongoDB collection name. """ def __init__(self, db, collection): self.db = db self.collection = collection def _has_key(self, key): return self.db[self.collection].count_documents({"_id": key}) > 0 def _delete(self, key): return self.db[self.collection].delete_one({"_id": key}) def _get(self, key): try: item = next(self.db[self.collection].find({"_id": key})) return pickle.loads(item["v"]) except StopIteration: raise KeyError(key) def _open(self, key): return BytesIO(self._get(key)) def _put(self, key, value): self.db[self.collection].update_one( {"_id": key}, {"$set": {"v": Binary(pickle.dumps(value))}}, upsert=True) return key def _put_file(self, key, file): return self._put(key, file.read()) def iter_keys(self, prefix=u""): for item in self.db[self.collection].find({"_id": {"$regex": '^' + re.escape(prefix)}}): yield item["_id"] simplekv-0.14.1/simplekv/db/sql.py000066400000000000000000000047721364363245700170460ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 from io import BytesIO from .._compat import imap, text_type from .. import KeyValueStore, CopyMixin from sqlalchemy import Table, Column, String, LargeBinary, select, exists class SQLAlchemyStore(KeyValueStore, CopyMixin): def __init__(self, bind, metadata, tablename): self.bind = bind self.table = Table( tablename, metadata, # 250 characters is the maximum key length that we guarantee can be # handled by any kind of backend Column('key', String(250), primary_key=True), Column('value', LargeBinary, nullable=False) ) def _has_key(self, key): return self.bind.execute( select([exists().where(self.table.c.key == key)]) ).scalar() def _delete(self, key): self.bind.execute( self.table.delete(self.table.c.key == key) ) def _get(self, key): rv = self.bind.execute( select([self.table.c.value], self.table.c.key == key).limit(1) ).scalar() if not rv: raise KeyError(key) return rv def _open(self, key): return BytesIO(self._get(key)) def _copy(self, source, dest): con = self.bind.connect() with con.begin(): data = self.bind.execute( select([self.table.c.value], self.table.c.key == source).limit(1) ).scalar() if not data: raise KeyError(source) # delete the potential existing previous key con.execute(self.table.delete(self.table.c.key == dest)) con.execute(self.table.insert({ 'key': dest, 'value': data, })) con.close() return dest def _put(self, key, data): con = self.bind.connect() with con.begin(): # delete the old con.execute(self.table.delete(self.table.c.key == key)) # insert new con.execute(self.table.insert({ 'key': key, 'value': data })) # commit happens here con.close() return key def _put_file(self, key, file): return self._put(key, file.read()) def iter_keys(self, prefix=u""): query = select([self.table.c.key]) if prefix != "": query = query.where(self.table.c.key.like(prefix + '%')) return imap(lambda v: text_type(v[0]), self.bind.execute(query)) simplekv-0.14.1/simplekv/decorator.py000066400000000000000000000120071364363245700176320ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 from ._compat import quote_plus, unquote_plus, text_type, binary_type class StoreDecorator(object): """Base class for store decorators. The default implementation will use :func:`getattr` to pass through all attribute/method requests to an underlying object stored as :attr:`_dstore`. It will also pass through the :attr:`__getattr__` and :attr:`__contains__` python special methods. """ def __init__(self, store): self._dstore = store def __getattr__(self, attr): return getattr(self._dstore, attr) def __contains__(self, *args, **kwargs): return self._dstore.__contains__(*args, **kwargs) def __iter__(self, *args, **kwargs): return self._dstore.__iter__(*args, **kwargs) class KeyTransformingDecorator(StoreDecorator): # currently undocumented (== not advertised as a feature) def _map_key(self, key): return key def _map_key_prefix(self, key_prefix): return key_prefix def _unmap_key(self, key): return key def _filter(self, key): return True def __contains__(self, key): return self._map_key(key) in self._dstore def __iter__(self): return self.iter_keys() def delete(self, key): return self._dstore.delete(self._map_key(key)) def get(self, key, *args, **kwargs): return self._dstore.get(self._map_key(key), *args, **kwargs) def get_file(self, key, *args, **kwargs): return self._dstore.get_file(self._map_key(key), *args, **kwargs) def iter_keys(self, prefix=u""): return (self._unmap_key(k) for k in self._dstore.iter_keys(self._map_key_prefix(prefix)) if self._filter(k)) def iter_prefixes(self, delimiter, prefix=u""): dlen = len(delimiter) plen = len(prefix) memory = set() for k in self.iter_keys(prefix): pos = k.find(delimiter, plen) if pos >= 0: k = k[: pos + dlen] if k not in memory: yield k memory.add(k) def keys(self, prefix=u""): """Return a list of keys currently in store, in any order :raises IOError: If there was an error accessing the store. """ return list(self.iter_keys(prefix)) def open(self, key): return self._dstore.open(self._map_key(key)) def put(self, key, *args, **kwargs): return self._unmap_key( self._dstore.put(self._map_key(key), *args, **kwargs)) def put_file(self, key, *args, **kwargs): return self._unmap_key( self._dstore.put_file(self._map_key(key), *args, **kwargs)) # support for UrlMixin def url_for(self, key, *args, **kwargs): return self._dstore.url_for(self._map_key(key), *args, **kwargs) # support for CopyMixin def copy(self, source, dest): return self._dstore.copy(self._map_key(source), self._map_key(dest)) class PrefixDecorator(KeyTransformingDecorator): """Prefixes any key with a string before passing it on the decorated store. Automatically strips the prefix upon key retrieval. :param store: The store to pass keys on to. :param prefix: Prefix to add. """ def __init__(self, prefix, store): super(PrefixDecorator, self).__init__(store) self.prefix = prefix def _filter(self, key): return key.startswith(self.prefix) def _map_key(self, key): self._check_valid_key(key) return self.prefix + key def _map_key_prefix(self, key_prefix): return self.prefix + key_prefix def _unmap_key(self, key): assert key.startswith(self.prefix) return key[len(self.prefix):] class URLEncodeKeysDecorator(KeyTransformingDecorator): """URL-encodes keys before passing them on to the underlying store.""" def _map_key(self, key): if not isinstance(key, text_type): raise ValueError('%r is not a unicode string' % key) quoted = quote_plus(key.encode('utf-8')) if isinstance(quoted, binary_type): quoted = quoted.decode('utf-8') return quoted def _map_key_prefix(self, key_prefix): return self._map_key(key_prefix) def _unmap_key(self, key): return unquote_plus(key) class ReadOnlyDecorator(StoreDecorator): """ A read-only view of an underlying simplekv store Provides only access to the following methods/attributes of the underlying store: get, iter_keys, keys, open, get_file. It also forwards __contains__. Accessing any other method will raise AttributeError. Note that the original store for r/w can still be accessed, so using this class as a wrapper only provides protection against bugs and other kinds of unintentional writes; it is not meant to be a real security measure. """ def __getattr__(self, attr): if attr in ('get', 'iter_keys', 'keys', 'open', 'get_file'): return super(ReadOnlyDecorator, self).__getattr__(attr) else: raise AttributeError simplekv-0.14.1/simplekv/fs.py000066400000000000000000000173221364363245700162650ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 import os import os.path import shutil from . import KeyValueStore, UrlMixin, CopyMixin from ._compat import url_quote, text_type class FilesystemStore(KeyValueStore, UrlMixin, CopyMixin): """Store data in files on the filesystem. The *FilesystemStore* stores every value as its own file on the filesystem, all under a common directory. Any call to :meth:`.url_for` will result in a `file://`-URL pointing towards the internal storage to be generated. """ def __init__(self, root, perm=None, **kwargs): """Initialize new FilesystemStore When files are created, they will receive permissions depending on the current umask if *perm* is `None`. Otherwise, permissions are set expliicitly. Note that when using :func:`put_file` with a filename, an attempt to move the file will be made. Permissions and ownership of the file will be preserved that way. If *perm* is set, permissions will be changed. :param root: the base directory for the store :param perm: the permissions for files in the filesystem store """ super(FilesystemStore, self).__init__(**kwargs) self.root = text_type(root) self.perm = perm self.bufsize = 1024 * 1024 # 1m def _remove_empty_parents(self, path): parents = os.path.relpath(path, os.path.abspath(self.root)) while len(parents) > 0: absparent = os.path.join(self.root, parents) if os.path.isdir(absparent): if len(os.listdir(absparent)) == 0: os.rmdir(absparent) else: break parents = os.path.dirname(parents) def _build_filename(self, key): return os.path.abspath(os.path.join(self.root, key)) def _delete(self, key): try: targetname = self._build_filename(key) os.unlink(targetname) self._remove_empty_parents(targetname) except OSError as e: if not e.errno == 2: raise def _fix_permissions(self, filename): current_umask = os.umask(0) os.umask(current_umask) perm = self.perm if self.perm is None: perm = 0o666 & (0o777 ^ current_umask) os.chmod(filename, perm) def _has_key(self, key): return os.path.exists(self._build_filename(key)) def _open(self, key): try: f = open(self._build_filename(key), 'rb') return f except IOError as e: if 2 == e.errno: raise KeyError(key) else: raise def _copy(self, source, dest): try: source_file_name = self._build_filename(source) dest_file_name = self._build_filename(dest) self._ensure_dir_exists(os.path.dirname(dest_file_name)) shutil.copy(source_file_name, dest_file_name) self._fix_permissions(dest_file_name) return dest except IOError as e: if 2 == e.errno: raise KeyError(source) else: raise def _ensure_dir_exists(self, path): if not os.path.isdir(path): try: os.makedirs(path) except OSError as e: if not os.path.isdir(path): raise e def _put_file(self, key, file): bufsize = self.bufsize target = self._build_filename(key) self._ensure_dir_exists(os.path.dirname(target)) with open(target, 'wb') as f: while True: buf = file.read(bufsize) f.write(buf) if len(buf) < bufsize: break # when using umask, correct permissions are automatically applied # only chmod is necessary if self.perm is not None: self._fix_permissions(target) return key def _put_filename(self, key, filename): target = self._build_filename(key) self._ensure_dir_exists(os.path.dirname(target)) shutil.move(filename, target) # we do not know the permissions of the source file, rectify self._fix_permissions(target) return key def _url_for(self, key): full = os.path.abspath(self._build_filename(key)) parts = full.split(os.sep) location = '/'.join(url_quote(p, safe='') for p in parts) return 'file://' + location def keys(self, prefix=u""): root = os.path.abspath(self.root) result = [] for dp, dn, fn in os.walk(root): for f in fn: key = os.path.join(dp, f)[len(root) + 1:] if key.startswith(prefix): result.append(key) return result def iter_keys(self, prefix=u""): return iter(self.keys(prefix)) def iter_prefixes(self, delimiter, prefix=u""): if delimiter != os.sep: return super(FilesystemStore, self).iter_prefixes( delimiter, prefix, ) return self._iter_prefixes_efficient(delimiter, prefix) def _iter_prefixes_efficient(self, delimiter, prefix=u""): if delimiter in prefix: pos = prefix.rfind(delimiter) search_prefix = prefix[:pos] path = os.path.join(self.root, search_prefix) else: search_prefix = None path = self.root try: for k in os.listdir(path): subpath = os.path.join(path, k) if search_prefix is not None: k = os.path.join(search_prefix, k) if os.path.isdir(subpath): k += delimiter if k.startswith(prefix): yield k except OSError: # path does not exists pass class WebFilesystemStore(FilesystemStore): """FilesystemStore that supports generating URLs suitable for web applications. Most common use is to make the *root* directory of the filesystem store available through a webserver. Example: >>> from simplekv.fs import WebFilesystemStore >>> webserver_url_prefix = 'https://some.domain.invalid/files/' >>> webserver_root = '/var/www/some.domain.invalid/www-data/files/' >>> store = WebFilesystemStore(webserver_root, webserver_url_prefix) >>> print(store.url_for(u'some_key')) https://some.domain.invalid/files/some_key Note that the prefix is simply prepended to the relative URL for the key. It therefore, in most cases, must include a trailing slash. *url_prefix* may also be a callable, in which case it gets called with the filestore and key as an argument and should return an url_prefix. >>> from simplekv.fs import WebFilesystemStore >>> webserver_url_prefix = 'https://some.domain.invalid/files/' >>> webserver_root = '/var/www/some.domain.invalid/www-data/files/' >>> prefix_func = lambda store, key: webserver_url_prefix >>> store = WebFilesystemStore(webserver_root, prefix_func) >>> print(store.url_for(u'some_key')) https://some.domain.invalid/files/some_key """ def __init__(self, root, url_prefix, **kwargs): """Initialize new WebFilesystemStore. :param root: see :func:`simplekv.FilesystemStore.__init__` :param url_prefix: will get prepended to every url generated with url_for. """ super(WebFilesystemStore, self).__init__(root, **kwargs) self.url_prefix = url_prefix def _url_for(self, key): rel = key if callable(self.url_prefix): stem = self.url_prefix(self, key) else: stem = self.url_prefix return stem + url_quote(rel, safe='') simplekv-0.14.1/simplekv/gae.py000066400000000000000000000021141364363245700164020ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 from cStringIO import StringIO from google.appengine.ext import ndb from . import KeyValueStore class NdbStore(KeyValueStore): def __init__(self, obj_class): self.obj_class = obj_class def _delete(self, key): db_key = ndb.Key(self.obj_class, key) db_key.delete() def _get(self, key): obj = self.obj_class.get_by_id(id=key) if not obj: raise KeyError(key) return obj.v def _has_key(self, key): return self.obj_class.get_by_id(id=key) is not None def iter_keys(self, prefix=u""): qry_iter = self.obj_class.query().iter(keys_only=True) return filter(lambda k: k.string_id().startswith(prefix), (k.string_id() for k in qry_iter) ) def _open(self, key): return StringIO(self._get(key)) def _put(self, key, data): obj = self.obj_class(id=key, v=data) obj.put() return obj.key.string_id() def _put_file(self, key, file): return self._put(key, file.read()) simplekv-0.14.1/simplekv/git.py000066400000000000000000000143501364363245700164360ustar00rootroot00000000000000from io import BytesIO import re import time from dulwich.repo import Repo from dulwich.objects import Commit, Tree, Blob from . import KeyValueStore, __version__ from ._compat import text_type def _on_tree(repo, tree, components, obj): """Mounts an object on a tree, using the given path components. :param tree: Tree object to mount on. :param components: A list of strings of subpaths (i.e. ['foo', 'bar'] is equivalent to '/foo/bar') :param obj: Object to mount. If None, removes the object found at path and prunes the tree downwards. :return: A list of new entities that need to be added to the object store, where the last one is the new tree. """ # pattern-matching: if len(components) == 1: if isinstance(obj, Blob): mode = 0o100644 elif isinstance(obj, Tree): mode = 0o040000 elif obj is None: mode = None else: raise TypeError('Can only mount Blobs or Trees') name = components[0] if mode is not None: tree[name] = mode, obj.id return [tree] if name in tree: del tree[name] return [tree] elif len(components) > 1: a, bc = components[0], components[1:] if a in tree: a_tree = repo[tree[a][1]] if not isinstance(a_tree, Tree): a_tree = Tree() else: a_tree = Tree() res = _on_tree(repo, a_tree, bc, obj) a_tree_new = res[-1] if a_tree_new.items(): tree[a] = 0o040000, a_tree_new.id return res + [tree] # tree is empty if a in tree: del tree[a] return [tree] else: raise ValueError('Components can\'t be empty.') class GitCommitStore(KeyValueStore): AUTHOR = 'GitCommitStore (simplekv {}) <>'.format(__version__) TIMEZONE = None def __init__(self, repo_path, branch=b'master', subdir=b''): self.repo = Repo(repo_path) self.branch = branch # cleans up subdir, to a form of 'a/b/c' (no duplicate, leading or # trailing slashes) self.subdir = re.sub('#/+#', '/', subdir.decode('ascii').strip('/')) @property def _subdir_components(self): return [c.encode('ascii') for c in self.subdir.split('/')] def _key_components(self, key): return [c.encode('ascii') for c in key.split('/')] @property def _refname(self): return b'refs/heads/' + self.branch def _create_top_commit(self): # get the top commit, create empty one if it does not exist commit = Commit() # commit metadata author = self.AUTHOR.encode('utf8') commit.author = commit.committer = author commit.commit_time = commit.author_time = int(time.time()) if self.TIMEZONE is not None: tz = self.TIMEZONE else: tz = time.timezone if (time.localtime().tm_isdst) else time.altzone commit.commit_timezone = commit.author_timezone = tz commit.encoding = b'UTF-8' return commit def _delete(self, key): try: commit = self.repo[self._refname] tree = self.repo[commit.tree] except KeyError: return # not-found key errors are ignored commit = self._create_top_commit() objects_to_add = [] components = self._key_components(key) if self.subdir: components = self._subdir_components + components res = _on_tree(self.repo, tree, components, None) objects_to_add.extend(res) tree = res[-1] commit.tree = tree.id commit.message = ( 'Deleted key {}'.format(self.subdir + '/' + key)).encode('utf8') objects_to_add.append(commit) for obj in objects_to_add: self.repo.object_store.add_object(obj) self.repo.refs[self._refname] = commit.id def _get(self, key): # might raise key errors, except block corrects param try: commit = self.repo[self._refname] tree = self.repo[commit.tree] fn = self.subdir + '/' + key _, blob_id = tree.lookup_path(self.repo.__getitem__, fn.encode('ascii')) blob = self.repo[blob_id] except KeyError: raise KeyError(key) return blob.data def iter_keys(self, prefix=u""): try: commit = self.repo[self._refname] tree = self.repo[commit.tree] if self.subdir: tree = self.repo[tree.lookup_path( self.repo.__getitem__, self.subdir.encode('ascii'))[1]] except KeyError: pass else: for o in self.repo.object_store\ .iter_tree_contents(tree.sha().hexdigest().encode('ascii')): if o.path.decode('ascii').startswith(prefix): yield o.path.decode('ascii') def _open(self, key): return BytesIO(self._get(key)) def _put_file(self, key, file): # FIXME: it may be worth to try to move large files directly into the # store here return self._put(key, file.read()) def _put(self, key, data): commit = self._create_top_commit() commit.message = ( 'Updated key {}'.format(self.subdir + '/' + key)).encode('utf8') blob = Blob.from_string(data) try: parent_commit = self.repo[self._refname] except KeyError: # branch does not exist, start with an empty tree tree = Tree() else: commit.parents = [parent_commit.id] tree = self.repo[parent_commit.tree] objects_to_add = [blob] components = self._key_components(key) if self.subdir: components = self._subdir_components + components res = _on_tree(self.repo, tree, components, blob) objects_to_add.extend(res) commit.tree = res[-1].id objects_to_add.append(commit) # add objects for obj in objects_to_add: self.repo.object_store.add_object(obj) # update refs self.repo.refs[self._refname] = commit.id return key simplekv-0.14.1/simplekv/idgen.py000066400000000000000000000100231364363245700167320ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 """ In cases where you want to generate IDs automatically, decorators are available. These should be the outermost decorators, as they change the signature of some of the put methods slightly. >>> from simplekv.memory import DictStore >>> from simplekv.idgen import HashDecorator >>> >>> store = HashDecorator(DictStore()) >>> >>> key = store.put(None, b'my_data') # note the passing of 'None' as key >>> print(key) ab0c15b6029fdffce16b393f2d27ca839a76249e """ import hashlib import os import tempfile import uuid from .decorator import StoreDecorator from ._compat import text_type class HashDecorator(StoreDecorator): """Hash function decorator Overrides :meth:`.KeyValueStore.put` and :meth:`.KeyValueStore.put_file`. If a key of *None* is passed, the data/file is hashed using ``hashfunc``, which defaults to *hashlib.sha1*. """ def __init__(self, decorated_store, hashfunc=hashlib.sha1, template=u'{}'): self.hashfunc = hashfunc self._template = template super(HashDecorator, self).__init__(decorated_store) def put(self, key, data, *args, **kwargs): if not key: key = self._template.format(self.hashfunc(data).hexdigest()) return self._dstore.put(key, data, *args, **kwargs) def put_file(self, key, file, *args, **kwargs): bufsize = 1024 * 1024 phash = self.hashfunc() if not key: if isinstance(file, str): with open(file, 'rb') as source: while True: buf = source.read(bufsize) phash.update(buf) if len(buf) < bufsize: break return self._dstore.put_file( self._template.format(phash.hexdigest()), file, *args, **kwargs) else: tmpfile = tempfile.NamedTemporaryFile(delete=False) try: while True: buf = file.read(bufsize) phash.update(buf) tmpfile.write(buf) if len(buf) < bufsize: break tmpfile.close() return self._dstore.put_file( self._template.format(phash.hexdigest()), tmpfile.name, *args, **kwargs ) finally: try: os.unlink(tmpfile.name) except OSError as e: if 2 == e.errno: pass # file already gone else: raise return self._dstore.put_file(key, file, *args, **kwargs) class UUIDDecorator(StoreDecorator): """UUID generating decorator Overrides :meth:`.KeyValueStore.put` and :meth:`.KeyValueStore.put_file`. If a key of *None* is passed, a new UUID will be generated as the key. The attribute `uuidfunc` determines which UUID-function to use and defaults to 'uuid1'. .. note:: There seems to be a bug in the uuid module that prevents initializing `uuidfunc` too early. For that reason, it is a string that will be looked up using :func:`getattr` on the :mod:`uuid` module. """ # for strange reasons, this needs to be looked up as late as possible uuidfunc = 'uuid1' def __init__(self, store, template=u'{}'): super(UUIDDecorator, self).__init__(store) self._template = template def put(self, key, data, *args, **kwargs): if not key: key = text_type(getattr(uuid, self.uuidfunc)()) return self._dstore.put( self._template.format(key), data, *args, **kwargs ) def put_file(self, key, file, *args, **kwargs): if not key: key = text_type(getattr(uuid, self.uuidfunc)()) return self._dstore.put_file( self._template.format(key), file, *args, **kwargs ) simplekv-0.14.1/simplekv/memory/000077500000000000000000000000001364363245700166065ustar00rootroot00000000000000simplekv-0.14.1/simplekv/memory/__init__.py000066400000000000000000000015711364363245700207230ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 from io import BytesIO from .._compat import ifilter from .. import KeyValueStore, CopyMixin class DictStore(KeyValueStore, CopyMixin): """Store data in a dictionary. This store uses a dictionary as the backend for storing, its implementation is straightforward. The dictionary containing all data is available as `d`. """ def __init__(self, d=None): self.d = d or {} def _delete(self, key): self.d.pop(key, None) def _has_key(self, key): return key in self.d def _open(self, key): return BytesIO(self.d[key]) def _copy(self, source, dest): self.d[dest] = self.d[source] def _put_file(self, key, file): self.d[key] = file.read() return key def iter_keys(self, prefix=u""): return ifilter(lambda k: k.startswith(prefix), iter(self.d)) simplekv-0.14.1/simplekv/memory/redisstore.py000066400000000000000000000034411364363245700213450ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8 -*- from io import BytesIO from .. import KeyValueStore, TimeToLiveMixin, NOT_SET, FOREVER import re class RedisStore(TimeToLiveMixin, KeyValueStore): """Uses a redis-database as the backend. :param redis: An instance of :py:class:`redis.StrictRedis`. """ def __init__(self, redis): self.redis = redis def _delete(self, key): return self.redis.delete(key) def keys(self, prefix=u""): return list(map(lambda b: b.decode(), self.redis.keys(pattern=re.escape(prefix) + '*'))) def iter_keys(self, prefix=u""): return iter(self.keys(prefix)) def _has_key(self, key): return self.redis.exists(key) def _get(self, key): val = self.redis.get(key) if val is None: raise KeyError(key) return val def _get_file(self, key, file): file.write(self._get(key)) def _open(self, key): return BytesIO(self._get(key)) def _put(self, key, value, ttl_secs): if ttl_secs in (NOT_SET, FOREVER): # if we do not care about ttl, just use set # in redis, using SET will also clear the timeout # note that this assumes that there is no way in redis # to set a default timeout on keys self.redis.set(key, value) else: ittl = None try: ittl = int(ttl_secs) except ValueError: pass # let it blow up further down if ittl == ttl_secs: self.redis.setex(key, ittl, value) else: self.redis.psetex(key, int(ttl_secs * 1000), value) return key def _put_file(self, key, file, ttl_secs): self._put(key, file.read(), ttl_secs) return key simplekv-0.14.1/simplekv/net/000077500000000000000000000000001364363245700160645ustar00rootroot00000000000000simplekv-0.14.1/simplekv/net/__init__.py000066400000000000000000000000001364363245700201630ustar00rootroot00000000000000simplekv-0.14.1/simplekv/net/_azurestore_common.py000066400000000000000000000032341364363245700223520ustar00rootroot00000000000000""" Internal utilities for aztorestore_old and azurestore_new """ import hashlib import base64 LAZY_PROPERTY_ATTR_PREFIX = "_lazy_" def lazy_property(fn): """Decorator that makes a property lazy-evaluated. On first access, lazy properties are computed and saved as instance attribute with the name `'_lazy_' + method_name` Any subsequent property access then returns the cached value.""" attr_name = LAZY_PROPERTY_ATTR_PREFIX + fn.__name__ @property def _lazy_property(self): if not hasattr(self, attr_name): setattr(self, attr_name, fn(self)) return getattr(self, attr_name) return _lazy_property def _file_md5(file_, b64encode=True): """ Compute the md5 digest of a file in base64 encoding. For ``b64encode``, returns the base64 encoded string; otherwise, returns the bytes directly. """ md5 = hashlib.md5() chunk_size = 128 * md5.block_size for chunk in iter(lambda: file_.read(chunk_size), b""): md5.update(chunk) file_.seek(0) byte_digest = md5.digest() if b64encode: return base64.b64encode(byte_digest).decode() else: return byte_digest def _filename_md5(filename, b64encode=True): """ Compute the md5 digest of a file in base64 encoding. """ with open(filename, "rb") as f: return _file_md5(f, b64encode=b64encode) def _byte_buffer_md5(buffer_, b64encode=True): """ Computes the md5 digest of a byte buffer in base64 encoding. """ md5 = hashlib.md5(buffer_) byte_digest = md5.digest() if b64encode: return base64.b64encode(byte_digest).decode() else: return byte_digest simplekv-0.14.1/simplekv/net/_azurestore_new.py000066400000000000000000000210071364363245700216510ustar00rootroot00000000000000""" This implements the AzureBlockBlobStore for `azure-storage-blob~=12` """ import io from contextlib import contextmanager from .._compat import PY2 from .. import KeyValueStore from ._azurestore_common import ( _byte_buffer_md5, _file_md5, lazy_property, LAZY_PROPERTY_ATTR_PREFIX, ) if PY2: def _blobname_to_texttype(name): """ Convert the str `name` to unicode """ return name.decode('utf-8') else: def _blobname_to_texttype(name): return name @contextmanager def map_azure_exceptions(key=None, error_codes_pass=()): """Map Azure-specific exceptions to the simplekv-API.""" from azure.core.exceptions import AzureError try: yield except AzureError as ex: error_code = getattr(ex, "error_code", None) if error_code is not None and error_code in error_codes_pass: return if error_code == "BlobNotFound": raise KeyError(key) raise IOError(str(ex)) class AzureBlockBlobStore(KeyValueStore): def __init__( self, conn_string=None, container=None, public=False, create_if_missing=True, max_connections=2, max_block_size=None, max_single_put_size=None, checksum=False, socket_timeout=None, ): """ Note that socket_timeout is unused; it only exist for backward compatibility. """ self.conn_string = conn_string self.container = container self.public = public self.create_if_missing = create_if_missing self.max_connections = max_connections self.max_block_size = max_block_size self.max_single_put_size = max_single_put_size self.checksum = checksum # Using @lazy_property will (re-)create block_blob_service instance needed. # Together with the __getstate__ implementation below, this allows # AzureBlockBlobStore to be pickled, even if # azure.storage.blob.BlockBlobService does not support pickling. @lazy_property def blob_container_client(self): from azure.storage.blob import BlobServiceClient kwargs = {} if self.max_single_put_size: kwargs["max_single_put_size"] = self.max_single_put_size if self.max_block_size: kwargs["max_block_size"] = self.max_block_size service_client = BlobServiceClient.from_connection_string( self.conn_string, **kwargs ) container_client = service_client.get_container_client(self.container) if self.create_if_missing: with map_azure_exceptions(error_codes_pass=("ContainerAlreadyExists")): container_client.create_container( public_access="container" if self.public else None ) return container_client def _delete(self, key): with map_azure_exceptions(key, error_codes_pass=("BlobNotFound",)): self.blob_container_client.delete_blob(key) def _get(self, key): with map_azure_exceptions(key): blob_client = self.blob_container_client.get_blob_client(key) downloader = blob_client.download_blob(max_concurrency=self.max_connections) return downloader.readall() def _has_key(self, key): blob_client = self.blob_container_client.get_blob_client(key) with map_azure_exceptions(key, ("BlobNotFound",)): blob_client.get_blob_properties() return True return False def iter_keys(self, prefix=None): with map_azure_exceptions(): blobs = self.blob_container_client.list_blobs(name_starts_with=prefix) def gen_names(): with map_azure_exceptions(): for blob in blobs: yield _blobname_to_texttype(blob.name) return gen_names() def iter_prefixes(self, delimiter, prefix=u""): return ( _blobname_to_texttype(blob_prefix.name) for blob_prefix in self.blob_container_client.walk_blobs( name_starts_with=prefix, delimiter=delimiter ) ) def _open(self, key): with map_azure_exceptions(key): blob_client = self.blob_container_client.get_blob_client(key) return IOInterface(blob_client, self.max_connections) def _put(self, key, data): from azure.storage.blob import ContentSettings if self.checksum: content_settings = ContentSettings( content_md5=_byte_buffer_md5(data, b64encode=False) ) else: content_settings = ContentSettings() with map_azure_exceptions(key): blob_client = self.blob_container_client.get_blob_client(key) blob_client.upload_blob( data, overwrite=True, content_settings=content_settings, max_concurrency=self.max_connections, ) return key def _put_file(self, key, file): from azure.storage.blob import ContentSettings if self.checksum: content_settings = ContentSettings(content_md5=_file_md5(file, b64encode=False)) else: content_settings = ContentSettings() with map_azure_exceptions(key): blob_client = self.blob_container_client.get_blob_client(key) blob_client.upload_blob( file, overwrite=True, content_settings=content_settings, max_concurrency=self.max_connections, ) return key def _get_file(self, key, file): with map_azure_exceptions(key): blob_client = self.blob_container_client.get_blob_client(key) downloader = blob_client.download_blob(max_concurrency=self.max_connections) downloader.readinto(file) def __getstate__(self): # keep all of __dict__, except lazy properties: return { key: value for key, value in self.__dict__.items() if not key.startswith(LAZY_PROPERTY_ATTR_PREFIX) } class IOInterface(io.BufferedIOBase): """ Class which provides a file-like interface to selectively read from a blob in the blob store. """ def __init__(self, blob_client, max_connections): super(IOInterface, self).__init__() self.blob_client = blob_client self.max_connections = max_connections blob_props = self.blob_client.get_blob_properties() self.size = blob_props.size self.pos = 0 def tell(self): """Returns he current offset as int. Always >= 0.""" if self.closed: raise ValueError("I/O operation on closed file") return self.pos def read(self, size=-1): """Returns 'size' amount of bytes or less if there is no more data. If no size is given all data is returned. size can be >= 0.""" if self.closed: raise ValueError("I/O operation on closed file") max_size = max(0, self.size - self.pos) if size < 0 or size > max_size: size = max_size if size == 0: return b"" downloader = self.blob_client.download_blob( self.pos, size, max_concurrency=self.max_connections ) b = downloader.readall() self.pos += len(b) return b def seek(self, offset, whence=0): """Move to a new offset either relative or absolute. whence=0 is absolute, whence=1 is relative, whence=2 is relative to the end. Any relative or absolute seek operation which would result in a negative position is undefined and that case can be ignored in the implementation. Any seek operation which moves the position after the stream should succeed. tell() should report that position and read() should return an empty bytes object.""" if self.closed: raise ValueError("I/O operation on closed file") if whence == 0: if offset < 0: raise IOError("seek would move position outside the file") self.pos = offset elif whence == 1: if self.pos + offset < 0: raise IOError("seek would move position outside the file") self.pos += offset elif whence == 2: if self.size + offset < 0: raise IOError("seek would move position outside the file") self.pos = self.size + offset return self.pos def seekable(self): return True def readable(self): return True simplekv-0.14.1/simplekv/net/_azurestore_old.py000066400000000000000000000234471364363245700216500ustar00rootroot00000000000000""" This implements the AzureBlockBlobStore for `azure-storage-blob<12` """ import io from contextlib import contextmanager from ._azurestore_common import _byte_buffer_md5, _file_md5, _filename_md5,\ lazy_property, LAZY_PROPERTY_ATTR_PREFIX from .._compat import binary_type from .. import KeyValueStore @contextmanager def map_azure_exceptions(key=None, exc_pass=()): """Map Azure-specific exceptions to the simplekv-API.""" from azure.common import AzureMissingResourceHttpError, AzureHttpError,\ AzureException try: yield except AzureMissingResourceHttpError as ex: if ex.__class__.__name__ not in exc_pass: s = str(ex) if s.startswith(u"The specified container does not exist."): raise IOError(s) raise KeyError(key) except AzureHttpError as ex: if ex.__class__.__name__ not in exc_pass: raise IOError(str(ex)) except AzureException as ex: if ex.__class__.__name__ not in exc_pass: raise IOError(str(ex)) class AzureBlockBlobStore(KeyValueStore): def __init__(self, conn_string=None, container=None, public=False, create_if_missing=True, max_connections=2, max_block_size=None, max_single_put_size=None, checksum=False, socket_timeout=None): self.conn_string = conn_string self.container = container self.public = public self.create_if_missing = create_if_missing self.max_connections = max_connections self.max_single_put_size = max_single_put_size self.max_block_size = max_block_size self.checksum = checksum self.socket_timeout = socket_timeout # Using @lazy_property will (re-)create block_blob_service instance needed. # Together with the __getstate__ implementation below, this allows # AzureBlockBlobStore to be pickled, even if # azure.storage.blob.BlockBlobService does not support pickling. @lazy_property def block_blob_service(self): from azure.storage.blob import BlockBlobService, PublicAccess block_blob_service = BlockBlobService( connection_string=self.conn_string, socket_timeout=self.socket_timeout, ) if self.max_block_size is not None: block_blob_service.MAX_BLOCK_SIZE = self.max_block_size if self.max_block_size is not None: block_blob_service.MAX_SINGLE_PUT_SIZE = self.max_single_put_size if self.create_if_missing: block_blob_service.create_container( self.container, public_access=PublicAccess.Container if self.public else None ) return block_blob_service def _delete(self, key): with map_azure_exceptions(key=key, exc_pass=['AzureMissingResourceHttpError']): self.block_blob_service.delete_blob(self.container, key) def _get(self, key): with map_azure_exceptions(key=key): return self.block_blob_service.get_blob_to_bytes( container_name=self.container, blob_name=key, max_connections=self.max_connections, ).content def _has_key(self, key): with map_azure_exceptions(key=key): return self.block_blob_service.exists(self.container, key) def iter_keys(self, prefix=u""): if prefix == "": prefix = None with map_azure_exceptions(): blobs = self.block_blob_service.list_blob_names(self.container, prefix=prefix) return (blob.decode('utf-8') if isinstance(blob, binary_type) else blob for blob in blobs) def iter_prefixes(self, delimiter, prefix=u""): if prefix == "": prefix = None with map_azure_exceptions(): blobs = self.block_blob_service.list_blob_names( self.container, prefix=prefix, delimiter=delimiter ) return (blob.decode('utf-8') if isinstance(blob, binary_type) else blob for blob in blobs) def _open(self, key): with map_azure_exceptions(key=key): return IOInterface(self.block_blob_service, self.container, key, self.max_connections) def _put(self, key, data): from azure.storage.blob.models import ContentSettings if self.checksum: content_settings = ContentSettings(content_md5=_byte_buffer_md5(data)) else: content_settings = ContentSettings() with map_azure_exceptions(key=key): self.block_blob_service.create_blob_from_bytes( container_name=self.container, blob_name=key, blob=data, max_connections=self.max_connections, content_settings=content_settings, ) return key def _put_file(self, key, file): from azure.storage.blob.models import ContentSettings if self.checksum: content_settings = ContentSettings(content_md5=_file_md5(file)) else: content_settings = ContentSettings() with map_azure_exceptions(key=key): self.block_blob_service.create_blob_from_stream( container_name=self.container, blob_name=key, stream=file, max_connections=self.max_connections, content_settings=content_settings, ) return key def _get_file(self, key, file): with map_azure_exceptions(key=key): self.block_blob_service.get_blob_to_stream( container_name=self.container, blob_name=key, stream=file, max_connections=self.max_connections, ) def _get_filename(self, key, filename): with map_azure_exceptions(key=key): self.block_blob_service.get_blob_to_path( container_name=self.container, blob_name=key, file_path=filename, max_connections=self.max_connections, ) def _put_filename(self, key, filename): from azure.storage.blob.models import ContentSettings if self.checksum: content_settings = ContentSettings(content_md5=_filename_md5(filename)) else: content_settings = ContentSettings() with map_azure_exceptions(key=key): self.block_blob_service.create_blob_from_path( container_name=self.container, blob_name=key, file_path=filename, max_connections=self.max_connections, content_settings=content_settings, ) return key def __getstate__(self): # keep all of __dict__, except lazy properties: return { key: value for key, value in self.__dict__.items() if not key.startswith(LAZY_PROPERTY_ATTR_PREFIX) } class IOInterface(io.BufferedIOBase): """ Class which provides a file-like interface to selectively read from a blob in the blob store. """ def __init__(self, block_blob_service, container_name, key, max_connections): super(IOInterface, self).__init__() self.block_blob_service = block_blob_service self.container_name = container_name self.key = key self.max_connections = max_connections blob = self.block_blob_service.get_blob_properties(container_name, key) self.size = blob.properties.content_length self.pos = 0 def tell(self): """Returns he current offset as int. Always >= 0.""" if self.closed: raise ValueError("I/O operation on closed file") return self.pos def read(self, size=-1): """Returns 'size' amount of bytes or less if there is no more data. If no size is given all data is returned. size can be >= 0.""" if self.closed: raise ValueError("I/O operation on closed file") with map_azure_exceptions(key=self.key): if size < 0: size = self.size - self.pos end = min(self.pos + size - 1, self.size - 1) if self.pos > end: return b'' b = self.block_blob_service.get_blob_to_bytes( container_name=self.container_name, blob_name=self.key, start_range=self.pos, end_range=end, # end_range is inclusive max_connections=self.max_connections, ) self.pos += len(b.content) return b.content def seek(self, offset, whence=0): """Move to a new offset either relative or absolute. whence=0 is absolute, whence=1 is relative, whence=2 is relative to the end. Any relative or absolute seek operation which would result in a negative position is undefined and that case can be ignored in the implementation. Any seek operation which moves the position after the stream should succeed. tell() should report that position and read() should return an empty bytes object.""" if self.closed: raise ValueError("I/O operation on closed file") if whence == 0: if offset < 0: raise IOError('seek would move position outside the file') self.pos = offset elif whence == 1: if self.pos + offset < 0: raise IOError('seek would move position outside the file') self.pos += offset elif whence == 2: if self.size + offset < 0: raise IOError('seek would move position outside the file') self.pos = self.size + offset return self.pos def seekable(self): return True def readable(self): return True simplekv-0.14.1/simplekv/net/azurestore.py000066400000000000000000000003671364363245700206470ustar00rootroot00000000000000# coding=utf8 try: from azure.storage.blob import BlockBlobService # noqa: F401 from ._azurestore_old import AzureBlockBlobStore except ImportError: from ._azurestore_new import AzureBlockBlobStore __all__ = ["AzureBlockBlobStore"] simplekv-0.14.1/simplekv/net/botostore.py000066400000000000000000000110331364363245700204540ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 import os from .._compat import imap from .. import KeyValueStore, UrlMixin, CopyMixin from contextlib import contextmanager @contextmanager def map_boto_exceptions(key=None, exc_pass=()): """Map boto-specific exceptions to the simplekv-API.""" from boto.exception import BotoClientError, BotoServerError, \ StorageResponseError try: yield except StorageResponseError as e: if e.code == 'NoSuchKey': raise KeyError(key) raise IOError(str(e)) except (BotoClientError, BotoServerError) as e: if e.__class__.__name__ not in exc_pass: raise IOError(str(e)) class BotoStore(KeyValueStore, UrlMixin, CopyMixin): def __init__(self, bucket, prefix='', url_valid_time=0, reduced_redundancy=False, public=False, metadata=None): self.prefix = prefix.strip().lstrip('/') self.bucket = bucket self.reduced_redundancy = reduced_redundancy self.public = public self.url_valid_time = url_valid_time self.metadata = metadata or {} def __new_key(self, name): from boto.s3.key import Key k = Key(self.bucket, self.prefix + name) if self.metadata: k.update_metadata(self.metadata) return k def __upload_args(self): """Generates a dictionary of arguments to pass to various set_content_from* functions. This allows us to save API calls by passing the necessary parameters on with the upload.""" d = { 'reduced_redundancy': self.reduced_redundancy, } if self.public: d['policy'] = 'public-read' return d def iter_keys(self, prefix=u""): with map_boto_exceptions(): prefix_len = len(self.prefix) return imap(lambda k: k.name[prefix_len:], self.bucket.list(self.prefix + prefix)) def _has_key(self, key): with map_boto_exceptions(key=key): return bool(self.bucket.get_key(self.prefix + key)) def _delete(self, key): from boto.exception import StorageResponseError try: self.bucket.delete_key(self.prefix + key) except StorageResponseError as e: if e.code != 'NoSuchKey': raise IOError(str(e)) def _get(self, key): k = self.__new_key(key) with map_boto_exceptions(key=key): return k.get_contents_as_string() def _get_file(self, key, file): k = self.__new_key(key) with map_boto_exceptions(key=key): return k.get_contents_to_file(file) def _get_filename(self, key, filename): k = self.__new_key(key) with map_boto_exceptions(key=key): return k.get_contents_to_filename(filename) def _open(self, key): from boto.s3.keyfile import KeyFile class SimpleKeyFile(KeyFile): def read(self, size=-1): if self.closed: raise ValueError("I/O operation on closed file") if size < 0: size = self.key.size - self.location return KeyFile.read(self, size) def seekable(self): return False def readable(self): return True k = self.__new_key(key) with map_boto_exceptions(key=key): return SimpleKeyFile(k) def _copy(self, source, dest): if not self._has_key(source): raise KeyError(source) with map_boto_exceptions(key=source): self.bucket.copy_key(self.prefix + dest, self.bucket.name, self.prefix + source) def _put(self, key, data): k = self.__new_key(key) with map_boto_exceptions(key=key): k.set_contents_from_string( data, **self.__upload_args() ) return key def _put_file(self, key, file): k = self.__new_key(key) with map_boto_exceptions(key=key): k.set_contents_from_file( file, **self.__upload_args() ) return key def _put_filename(self, key, filename): k = self.__new_key(key) with map_boto_exceptions(key=key): k.set_contents_from_filename( filename, **self.__upload_args() ) return key def _url_for(self, key): k = self.__new_key(key) with map_boto_exceptions(key=key): return k.generate_url(expires_in=self.url_valid_time, query_auth=False) simplekv-0.14.1/tests/000077500000000000000000000000001364363245700146065ustar00rootroot00000000000000simplekv-0.14.1/tests/basic_store.py000066400000000000000000000326561364363245700174710ustar00rootroot00000000000000# coding: utf8 import os import time import tempfile import pytest from simplekv._compat import BytesIO, xrange, text_type from simplekv.decorator import PrefixDecorator from simplekv.crypt import HMACDecorator from simplekv.idgen import UUIDDecorator, HashDecorator from simplekv import CopyMixin class BasicStore(object): def test_store(self, store, key, value): key = store.put(key, value) assert isinstance(key, text_type) def test_unicode_store(self, store, key, unicode_value): with pytest.raises(IOError): store.put(key, unicode_value) def test_store_and_retrieve(self, store, key, value): store.put(key, value) assert store.get(key) == value def test_store_and_retrieve_filelike(self, store, key, value): store.put_file(key, BytesIO(value)) assert store.get(key) == value def test_store_and_retrieve_overwrite(self, store, key, value, value2): store.put_file(key, BytesIO(value)) assert store.get(key) == value store.put(key, value2) assert store.get(key) == value2 def test_store_and_open(self, store, key, value): store.put_file(key, BytesIO(value)) assert store.open(key).read() == value def test_store_and_copy(self, store, key, key2, value): if not isinstance(store, CopyMixin): pytest.skip() store.put(key, value) assert store.get(key) == value store.copy(key, key2) assert store.get(key) == value assert store.get(key2) == value def test_store_and_copy_overwrite(self, store, key, key2, value, value2): if not isinstance(store, CopyMixin): pytest.skip() store.put(key, value) store.put(key2, value2) assert store.get(key) == value assert store.get(key2) == value2 store.copy(key, key2) assert store.get(key) == value assert store.get(key2) == value def test_open_incremental_read(self, store, key, long_value): store.put_file(key, BytesIO(long_value)) ok = store.open(key) assert long_value[:3] == ok.read(3) assert long_value[3:5] == ok.read(2) assert long_value[5:8] == ok.read(3) def test_key_error_on_nonexistant_get(self, store, key): with pytest.raises(KeyError): store.get(key) def test_key_error_on_nonexistant_copy(self, store, key, key2): if not isinstance(store, CopyMixin): pytest.skip() with pytest.raises(KeyError): store.copy(key, key2) def test_key_error_on_nonexistant_open(self, store, key): with pytest.raises(KeyError): store.open(key) def test_key_error_on_nonexistant_get_file(self, store, key): with pytest.raises(KeyError): store.get_file(key, BytesIO()) def test_key_error_on_nonexistant_get_filename(self, store, key): with pytest.raises(KeyError): store.get_file(key, '/dev/null') def test_exception_on_invalid_key_get(self, store, invalid_key): with pytest.raises(ValueError): store.get(invalid_key) def test_exception_on_invalid_key_copy(self, store, invalid_key, key): if not isinstance(store, CopyMixin): pytest.skip() with pytest.raises(ValueError): store.copy(invalid_key, key) with pytest.raises(ValueError): store.copy(key, invalid_key) def test_exception_on_invalid_key_get_file(self, store, invalid_key): with pytest.raises(ValueError): store.get_file(invalid_key, '/dev/null') def test_exception_on_invalid_key_delete(self, store, invalid_key): with pytest.raises(ValueError): store.delete(invalid_key) def test_put_file(self, store, key, value): tmp = tempfile.NamedTemporaryFile(delete=False) try: tmp.write(value) tmp.close() store.put_file(key, tmp.name) assert store.get(key) == value finally: if os.path.exists(tmp.name): os.unlink(tmp.name) def test_put_opened_file(self, store, key, value): with tempfile.NamedTemporaryFile() as tmp: tmp.write(value) tmp.flush() store.put_file(key, open(tmp.name, 'rb')) assert store.get(key) == value def test_get_into_file(self, store, key, value, tmp_path): store.put(key, value) out_filename = os.path.join(str(tmp_path), 'output') store.get_file(key, out_filename) assert open(out_filename, 'rb').read() == value def test_get_into_stream(self, store, key, value): store.put(key, value) output = BytesIO() store.get_file(key, output) assert output.getvalue() == value def test_put_return_value(self, store, key, value): assert key == store.put(key, value) def test_put_file_return_value(self, store, key, value): assert key == store.put_file(key, BytesIO(value)) def test_put_filename_return_value(self, store, key, value): tmp = tempfile.NamedTemporaryFile(delete=False) try: tmp.write(value) tmp.close() assert key == store.put_file(key, tmp.name) finally: if os.path.exists(tmp.name): os.unlink(tmp.name) def test_delete(self, store, key, value): store.put(key, value) assert value == store.get(key) store.delete(key) with pytest.raises(KeyError): store.get(key) def test_multiple_delete_fails_without_error(self, store, key, value): store.put(key, value) store.delete(key) store.delete(key) store.delete(key) def test_can_delete_key_that_never_exists(self, store, key): store.delete(key) def test_key_iterator(self, store, key, key2, value, value2): store.put(key, value) store.put(key2, value2) l = [] for k in store.iter_keys(): assert isinstance(k, text_type) l.append(k) l.sort() assert l == sorted([key, key2]) def test_key_iterator_with_prefix(self, store, key, key2, value): prefix = key key_prefix_1 = prefix + '_key1' key_prefix_2 = prefix + '_key2' store.put(key_prefix_1, value) store.put(key_prefix_2, value) store.put(key2, value) l = [] for k in store.iter_keys(): l.append(k) l.sort() assert l == sorted([key_prefix_1, key_prefix_2, key2]) l = [] for k in store.iter_keys(prefix): l.append(k) l.sort() assert l == sorted([key_prefix_1, key_prefix_2]) def test_prefix_iterator(self, store, value): delimiter = u"X" for k in [ u"X", u"a1Xb1", u"a1Xb1", u"a2X", u"a2Xb1", u"a3", u"a4Xb1Xc1", u"a4Xb1Xc2", u"a4Xb2Xc1", u"a4Xb3", ]: store.put(k, value) l = sorted(store.iter_prefixes(u"X")) assert l == [u"X", u"a1X", u"a2X", u"a3", u"a4X"] l = sorted(store.iter_prefixes(u"X", prefix=u"a4X")) assert l == [u"a4Xb1X", u"a4Xb2X", u"a4Xb3"] l = sorted(store.iter_prefixes(u"X", prefix=u"foo")) assert l == [] def test_keys(self, store, key, key2, value, value2): store.put(key, value) store.put(key2, value2) l = store.keys() assert isinstance(l, list) for k in l: assert isinstance(k, text_type) assert set(l) == {key, key2} def test_keys_with_prefix(self, store, key, key2, value): prefix = key key_prefix_1 = prefix + '_key1' key_prefix_2 = prefix + '_key2' store.put(key_prefix_1, value) store.put(key_prefix_2, value) store.put(key2, value) l = store.keys() assert isinstance(l, list) assert set(l) == {key_prefix_1, key_prefix_2, key2} l = store.keys(prefix) assert isinstance(l, list) assert set(l) == {key_prefix_1, key_prefix_2} def test_has_key(self, store, key, key2, value): store.put(key, value) assert key in store assert key2 not in store def test_has_key_with_delete(self, store, key, value): assert key not in store store.put(key, value) assert key in store store.delete(key) assert key not in store store.put(key, value) assert key in store def test_get_with_delete(self, store, key, value): with pytest.raises(KeyError): store.get(key) store.put(key, value) store.get(key) store.delete(key) with pytest.raises(KeyError): store.get(key) store.put(key, value) store.get(key) def test_max_key_length(self, store, max_key, value): new_key = store.put(max_key, value) assert new_key == max_key assert value == store.get(max_key) def test_a_lot_of_puts(self, store, key, value): a_lot = 20 for i in xrange(a_lot): key = key + '_{}'.format(i) store.put(key, value) # small extra time added to account for variance TTL_MARGIN = 1 class TTLStore(object): @pytest.fixture def ustore(self, store): return UUIDDecorator(store) @pytest.fixture(params=['hash', 'uuid', 'hmac', 'prefix']) def dstore(self, request, store, secret_key, ustore): if request.param == 'hash': return HashDecorator(store) elif request.param == 'uuid': return ustore elif request.param == 'hmac': return HMACDecorator(secret_key, store) elif request.param == 'prefix': return PrefixDecorator('SaMpLe_PrEfIX', store) @pytest.fixture(params=[0.4, 1]) def small_ttl(self, request): return request.param def test_put_with_negative_ttl_throws_error(self, store, key, value): with pytest.raises(ValueError): store.put(key, value, ttl_secs=-1) def test_put_with_non_numeric_ttl_throws_error(self, store, key, value): with pytest.raises(ValueError): store.put(key, value, ttl_secs='badttl') def test_put_with_ttl_argument(self, store, key, value, small_ttl): store.put(key, value, small_ttl) time.sleep(small_ttl + TTL_MARGIN) with pytest.raises(KeyError): store.get(key) def test_put_set_default(self, store, key, value, small_ttl): store.default_ttl_secs = small_ttl store.put(key, value) time.sleep(small_ttl + TTL_MARGIN) with pytest.raises(KeyError): store.get(key) def test_put_file_with_ttl_argument(self, store, key, value, small_ttl): store.put_file(key, BytesIO(value), small_ttl) time.sleep(small_ttl + TTL_MARGIN) with pytest.raises(KeyError): store.get(key) def test_put_file_set_default(self, store, key, value, small_ttl): store.default_ttl_secs = small_ttl store.put_file(key, BytesIO(value)) time.sleep(small_ttl + TTL_MARGIN) with pytest.raises(KeyError): store.get(key) def test_uuid_decorator(self, ustore, value): key = ustore.put(None, value) assert key def test_advertises_ttl_features(self, store): assert store.ttl_support is True assert hasattr(store, 'ttl_support') assert getattr(store, 'ttl_support') is True def test_advertises_ttl_features_through_decorator(self, dstore): assert dstore.ttl_support is True assert hasattr(dstore, 'ttl_support') assert getattr(dstore, 'ttl_support') is True def test_can_pass_ttl_through_decorator(self, dstore, key, value): dstore.put(key, value, ttl_secs=10) class OpenSeekTellStore(object): def test_open_seek_and_tell_empty_value(self, store, key): value = b'' store.put(key, value) ok = store.open(key) assert ok.seekable() assert ok.seek(10) == 10 assert ok.tell() == 10 assert ok.seek(-6, 1) == 4 assert ok.tell() == 4 with pytest.raises(IOError): ok.seek(-1, 0) with pytest.raises(IOError): ok.seek(-6, 1) with pytest.raises(IOError): ok.seek(-1, 2) assert ok.tell() == 4 assert b'' == ok.read(1) def test_open_seek_and_tell(self, store, key, long_value): store.put(key, long_value) ok = store.open(key) assert ok.seekable() assert ok.readable() ok.seek(10) assert ok.tell() == 10 ok.seek(-6, 1) assert ok.tell() == 4 with pytest.raises(IOError): ok.seek(-1, 0) with pytest.raises(IOError): ok.seek(-6, 1) with pytest.raises(IOError): ok.seek(-len(long_value) - 1, 2) assert ok.tell() == 4 assert long_value[4:5] == ok.read(1) assert ok.tell() == 5 ok.seek(-1, 2) length_lv = len(long_value) assert long_value[length_lv - 1:length_lv] == ok.read(1) assert ok.tell() == length_lv ok.seek(length_lv + 10, 0) assert ok.tell() == length_lv + 10 assert b'' == ok.read() ok.close() with pytest.raises(ValueError): ok.tell() with pytest.raises(ValueError): ok.read(1) with pytest.raises(ValueError): ok.seek(10) simplekv-0.14.1/tests/bucket_manager.py000066400000000000000000000043711364363245700201340ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 from simplekv._compat import ConfigParser from contextlib import contextmanager from uuid import uuid4 as uuid import pytest boto = pytest.importorskip('boto') @contextmanager def boto_bucket(access_key, secret_key, host, connect_func='connect_s3', ordinary_calling_format=False, bucket_name=None, port=None, is_secure=True): if ordinary_calling_format: from boto.s3.connection import OrdinaryCallingFormat conn = getattr(boto, connect_func)(access_key, secret_key, host=host, calling_format=OrdinaryCallingFormat (), port=port, is_secure=is_secure) else: conn = getattr(boto, connect_func)(access_key, secret_key, host=host, port=port, is_secure=is_secure) name = bucket_name or 'testrun-bucket-{}'.format(uuid()) bucket = conn.create_bucket(name) yield bucket for key in bucket.list(): key.delete() bucket.delete() def load_boto_credentials(): # loaded from the same place tox.ini. here's a sample # # [my-s3] # access_key=foo # secret_key=bar # connect_func=connect_s3 # # [my-gs] # access_key=foo # secret_key=bar # connect_func=connect_gs cfg_fn = 'boto_credentials.ini' parser = ConfigParser({'host': 's3.amazonaws.com', 'is_secure': 'true', 'ordinary_calling_format': 'false', }) if not parser.read(cfg_fn): pytest.skip('file {} not found'.format(cfg_fn)) for section in parser.sections(): yield { 'access_key': parser.get(section, 'access_key'), 'secret_key': parser.get(section, 'secret_key'), 'connect_func': parser.get(section, 'connect_func'), 'host': parser.get(section, 'host'), 'is_secure': parser.getboolean(section, 'is_secure'), 'port': parser.getint(section, 'port'), 'ordinary_calling_format': parser.getboolean(section, 'ordinary_calling_format') } boto_credentials = list(load_boto_credentials()) simplekv-0.14.1/tests/conftest.py000066400000000000000000000042101364363245700170020ustar00rootroot00000000000000# coding: utf8 import hashlib import pytest @pytest.fixture(params=['sha1', 'sha256', 'md5']) def hashfunc(request): return getattr(hashlib, request.param) @pytest.fixture(params=[b'secret_key_a', b'\x12\x00\x12test']) def secret_key(request): return request.param # values are short payloads to store @pytest.fixture(params=[b'a_short_value', b'another_short_value']) def value(request): return request.param @pytest.fixture(params=[b'the_other_value', b'other_value_2']) def value2(request): return request.param @pytest.fixture(params=[u'the_other_value', u'othäöü_valਊਏਐਓਔਕਖਗue_2']) def unicode_value(request): return request.param @pytest.fixture(params=[b'a_long_value' * 4 * 1024]) def long_value(request): return request.param # keys are always strings. only ascii chars are allowed @pytest.fixture(params=[u'short_key', u'different_key', u"""'!"`#$%&'()+,-.<=>?@[]^_{}~'"""]) def key(request): return request.param @pytest.fixture(params=[u'key_number_2', u'and_again_a_second_key']) def key2(request): return request.param @pytest.fixture(params=[u'ä', u'/', u'\x00', u'*', u'', u'no whitespace allowed']) def invalid_key(request): return request.param @pytest.fixture(params=[b'short_key', b'different_key', b"""'!"`#$%&'()+,-.<=>?@[]^_{}~'"""]) def bytestring_key(request): return request.param # maximum length key @pytest.fixture(params=[u'a' * 250, u'b' * 250]) def max_key(request): return request.param # Test class to derive from to get test fixtures for the extended keyspace class ExtendedKeyspaceTests: @pytest.fixture(params=[u'short ke/y', u'short_key', u'spaces in key', u"""'!"`#$%&'()+,-.<=>?@[]^_{}~/'"""]) def key(self, request): return request.param @pytest.fixture(params=[u'key_number/2 with space', u'and_again_a_second_key']) def key2(self, request): return request.param @pytest.fixture(params=[u'ä', u'/', u'\x00', u'*', u'']) def invalid_key(self, request): return request.param simplekv-0.14.1/tests/idgens.py000066400000000000000000000102301364363245700164250ustar00rootroot00000000000000import os import re import tempfile import uuid from simplekv.idgen import UUIDDecorator, HashDecorator from simplekv._compat import text_type import pytest UUID_REGEXP = re.compile( r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' ) class IDGen(object): @pytest.fixture(params=[ u'constant', u'foo{}bar', u'{}.jpeg', u'prefix-{}.hello', u'justprefix{}', ]) def idgen_template(self, request): return request.param class UUIDGen(IDGen): @pytest.fixture def uuidstore(self, store): return UUIDDecorator(store) @pytest.fixture def templated_uuidstore(self, store, idgen_template): return UUIDDecorator(store, idgen_template) def test_put_generates_uuid_form(self, uuidstore, value): key = uuidstore.put(None, value) assert UUID_REGEXP.match(key) def test_put_file_generates_uuid_form(self, uuidstore): key = uuidstore.put_file(None, open('/dev/null', 'rb')) assert UUID_REGEXP.match(key) tmpfile = tempfile.NamedTemporaryFile(delete=False) try: tmpfile.close() key2 = uuidstore.put_file(None, tmpfile.name) assert UUID_REGEXP.match(key2) finally: if os.path.exists(tmpfile.name): os.unlink(tmpfile.name) def test_put_generates_valid_uuid(self, uuidstore, value): key = uuidstore.put(None, value) uuid.UUID(hex=key) def test_put_file_generates_valid_uuid(self, uuidstore): key = uuidstore.put_file(None, open('/dev/null', 'rb')) uuid.UUID(hex=key) tmpfile = tempfile.NamedTemporaryFile(delete=False) try: tmpfile.close() key2 = uuidstore.put_file(None, tmpfile.name) uuid.UUID(hex=key2) finally: if os.path.exists(tmpfile.name): os.unlink(tmpfile.name) def test_templates_work(self, templated_uuidstore, value, idgen_template): key = templated_uuidstore.put(None, value) # should not be a valid UUID assert not UUID_REGEXP.match(key) class HashGen(IDGen): @pytest.fixture def hashstore(self, store, hashfunc): return HashDecorator(store, hashfunc) @pytest.fixture def templated_hashstore(self, store, hashfunc, idgen_template): return HashDecorator(store, hashfunc, idgen_template) @pytest.fixture def validate_hash(self, hashfunc): hash_regexp = re.compile(r'^[0-9a-f]{{{}}}$'.format( hashfunc().digest_size * 2, )) return hash_regexp.match @pytest.fixture def value_hash(self, hashfunc, value): return hashfunc(value).hexdigest() def test_put_generates_valid_form(self, hashstore, validate_hash, value): key = hashstore.put(None, value) assert validate_hash(key) def test_put_file_generates_valid_form(self, hashstore, validate_hash): key = hashstore.put_file(None, open('/dev/null', 'rb')) assert validate_hash(key) # this is not correct according to our interface # /dev/null cannot be claimed by the hashstore # key2 = hashstore.put_file(None, '/dev/null') # assert validate_hash(key2) def test_put_generates_correct_hash(self, hashstore, value_hash, value): key = hashstore.put(None, value) assert value_hash == key assert isinstance(key, text_type) def test_put_file_generates_correct_hash( self, hashstore, value_hash, value ): tmpfile = tempfile.NamedTemporaryFile(delete=False) try: tmpfile.write(value) tmpfile.close() with open(tmpfile.name, 'rb') as f: key = hashstore.put_file(None, f) assert key == value_hash key2 = hashstore.put_file(None, tmpfile.name) assert key2 == value_hash finally: if os.path.exists(tmpfile.name): os.unlink(tmpfile.name) def test_templates_work( self, templated_hashstore, value, idgen_template, value_hash ): key = templated_hashstore.put(None, value) assert idgen_template.format(value_hash) == key simplekv-0.14.1/tests/test_azure_store.py000066400000000000000000000214361364363245700205670ustar00rootroot00000000000000from uuid import uuid4 as uuid from simplekv._compat import ConfigParser, pickle from simplekv.net.azurestore import AzureBlockBlobStore from simplekv.contrib import ExtendedKeyspaceMixin from basic_store import BasicStore, OpenSeekTellStore from conftest import ExtendedKeyspaceTests import pytest from base64 import b64encode asb = pytest.importorskip('azure.storage.blob') def get_azure_conn_string(): cfg_fn = 'azure_credentials.ini' parser = ConfigParser({ 'protocol': 'https', 'endpoint': '', 'account_name': '', }) result = parser.read(cfg_fn) if not result: pytest.skip('file {} not found'.format(cfg_fn)) for section in parser.sections(): account_name = parser.get(section, 'account_name') if not account_name: pytest.skip("no 'account_name' found in file {}".format(cfg_fn)) account_key = parser.get(section, 'account_key') protocol = parser.get(section, 'protocol') endpoint = parser.get(section, 'endpoint') conn_string = 'DefaultEndpointsProtocol={};AccountName={};AccountKey={}'.format( protocol, account_name, account_key ) if endpoint: conn_string += ';BlobEndpoint={}'.format(endpoint) return conn_string def _delete_container(conn_string, container): try: # for azure-storage-blob>=12: from azure.storage.blob import BlobServiceClient from azure.core.exceptions import AzureError s = BlobServiceClient.from_connection_string(conn_string) try: s.delete_container(container) except AzureError as ex: # ignore the ContainerNotFound error: if ex.error_code != 'ContainerNotFound': raise except ImportError: # for azure-storage-blob<12 from azure.storage.blob import BlockBlobService s = BlockBlobService(connection_string=conn_string) s.delete_container(container) class TestAzureStorage(BasicStore, OpenSeekTellStore): @pytest.fixture def store(self): container = str(uuid()) conn_string = get_azure_conn_string() yield AzureBlockBlobStore(conn_string=conn_string, container=container, public=False) _delete_container(conn_string, container) class TestExtendedKeysAzureStorage(TestAzureStorage, ExtendedKeyspaceTests): @pytest.fixture def store(self): azure_storage_blob_major_version = int(asb.__version__.split('.', 1)[0]) conn_string = get_azure_conn_string() use_azurite = 'http://127.0.0.1:10000/devstoreaccount1' in conn_string if use_azurite and azure_storage_blob_major_version < 12: pytest.skip("Compatibility issues with azurite and azure-storage-blob<12") container = str(uuid()) class ExtendedKeysStore(ExtendedKeyspaceMixin, AzureBlockBlobStore): pass yield ExtendedKeysStore(conn_string=conn_string, container=container, public=False) _delete_container(conn_string, container) def test_azure_setgetstate(): container = str(uuid()) conn_string = get_azure_conn_string() store = AzureBlockBlobStore(conn_string=conn_string, container=container) store.put(u'key1', b'value1') buf = pickle.dumps(store, protocol=2) store = pickle.loads(buf) assert store.get(u'key1') == b'value1' _delete_container(conn_string, container) def test_azure_store_attributes(): abbs = AzureBlockBlobStore('CONN_STR', 'CONTAINER', max_connections=42, checksum=True) assert abbs.conn_string == 'CONN_STR' assert abbs.container == 'CONTAINER' assert abbs.public is False assert abbs.create_if_missing is True assert abbs.max_connections == 42 assert abbs.checksum is True abbs2 = pickle.loads(pickle.dumps(abbs)) assert abbs2.conn_string == 'CONN_STR' assert abbs2.container == 'CONTAINER' assert abbs2.public is False assert abbs2.create_if_missing is True assert abbs2.max_connections == 42 assert abbs2.checksum is True def test_azure_special_args(): # For azure-storage-blob 12, # test that the special arguments `max_block_size` and # `max_single_put_size` propagate to the constructed ContainerClient conn_string = get_azure_conn_string() MBS = 983645 MSP = 756235 abbs = AzureBlockBlobStore( conn_string=conn_string, container='container-unused', max_block_size=MBS, max_single_put_size=MSP, create_if_missing=False ) if hasattr(abbs, "blob_container_client"): cfg = abbs.blob_container_client._config assert cfg.max_single_put_size == MSP assert cfg.max_block_size == MBS class TestAzureExceptionHandling(object): def test_missing_container(self): container = str(uuid()) conn_string = get_azure_conn_string() store = AzureBlockBlobStore(conn_string=conn_string, container=container, create_if_missing=False) with pytest.raises(IOError) as exc: store.keys() assert u"The specified container does not exist." in str(exc.value) def test_wrong_endpoint(self): container = str(uuid()) conn_string = get_azure_conn_string() conn_settings = dict([s.split("=", 1) for s in conn_string.split(";") if s]) conn_settings['BlobEndpoint'] = 'https://host-does-not-exist/' conn_string = ';'.join('{}={}'.format(key, value) for key, value in conn_settings.items()) store = AzureBlockBlobStore(conn_string=conn_string, container=container, create_if_missing=False) if hasattr(store, 'block_blob_service'): from azure.storage.common.retry import ExponentialRetry store.block_blob_service.retry = ExponentialRetry( max_attempts=0 ).retry else: store.blob_container_client._config.retry_policy.total_retries = 0 with pytest.raises(IOError) as exc: store.put(u"key", b"data") assert u"connect" in str(exc.value) def test_wrong_credentials(self): container = str(uuid()) conn_string = \ 'DefaultEndpointsProtocol=https;AccountName={};AccountKey={}'.\ format("testaccount", "wrongsecret") store = AzureBlockBlobStore(conn_string=conn_string, container=container, create_if_missing=False) if hasattr(store, 'block_blob_service'): from azure.storage.common.retry import ExponentialRetry store.block_blob_service.retry = ExponentialRetry( max_attempts=0 ).retry else: store.blob_container_client._config.retry_policy.total_retries = 0 with pytest.raises(IOError) as exc: store.put(u"key", b"data") assert u"Incorrect padding" in str(exc.value) class TestChecksum(object): CONTENT = b'\1\2\3\4' EXPECTED_CHECKSUM = 'CNbAWiFRKnmh3+udKo8mLw==' KEY = 'testkey' @pytest.fixture def store(self): container = str(uuid()) conn_string = get_azure_conn_string() yield AzureBlockBlobStore( conn_string=conn_string, container=container, public=False, checksum=True, ) _delete_container(conn_string, container) def _checksum(self, store): # request the md5 checksum from azure and return the b64 encoded value if hasattr(store, 'block_blob_service'): return store.block_blob_service.get_blob_properties( store.container, self.KEY, ).properties.content_settings.content_md5 else: checksum_bytes = store.blob_container_client.get_blob_client( self.KEY ).get_blob_properties().content_settings.content_md5 return b64encode(checksum_bytes).decode() def test_checksum_put(self, store): store.put(self.KEY, self.CONTENT) assert self._checksum(store) == self.EXPECTED_CHECKSUM assert store.get(self.KEY) == self.CONTENT def test_checksum_put_file(self, store, tmpdir): file_ = tmpdir.join('my_file') file_.write(self.CONTENT) store.put_file(self.KEY, file_.open('rb')) assert self._checksum(store) == self.EXPECTED_CHECKSUM assert store.get(self.KEY) == self.CONTENT def test_checksum_put_filename(self, store, tmpdir): file_ = tmpdir.join('my_file') file_.write(self.CONTENT) store.put_file(self.KEY, str(file_)) assert self._checksum(store) == self.EXPECTED_CHECKSUM assert store.get(self.KEY) == self.CONTENT simplekv-0.14.1/tests/test_boto_store.py000066400000000000000000000056211364363245700204020ustar00rootroot00000000000000#!/usr/bin/env python import os import pytest boto = pytest.importorskip('boto') from simplekv.net.botostore import BotoStore from simplekv._compat import BytesIO from basic_store import BasicStore from url_store import UrlStore from bucket_manager import boto_credentials, boto_bucket from conftest import ExtendedKeyspaceTests from simplekv.contrib import ExtendedKeyspaceMixin @pytest.fixture(params=boto_credentials, ids=[c['access_key'] for c in boto_credentials]) def credentials(request): return request.param @pytest.yield_fixture() def bucket(credentials): with boto_bucket(**credentials) as bucket: yield bucket class TestBotoStorage(BasicStore, UrlStore): @pytest.fixture(params=[True, False]) def reduced_redundancy(self, request): return request.param @pytest.fixture def storage_class(self, reduced_redundancy): return 'REDUCED_REDUNDANCY' if reduced_redundancy else 'STANDARD' @pytest.fixture(params=['', '/test-prefix']) def prefix(self, request): return request.param @pytest.fixture def store(self, bucket, prefix, reduced_redundancy): return BotoStore(bucket, prefix, reduced_redundancy=reduced_redundancy) # Disable max key length test as it leads to problems with minio test_max_key_length = None def test_get_filename_nonexistant(self, store, key, tmp_path): # NOTE: boto misbehaves here and tries to erase the target file # the parent tests use /dev/null, which you really should not try # to os.remove! with pytest.raises(KeyError): store.get_file(key, os.path.join(str(tmp_path), 'a')) def test_key_error_on_nonexistant_get_filename(self, store, key, tmp_path): with pytest.raises(KeyError): store.get_file(key, os.path.join(str(tmp_path), 'a')) def test_storage_class_put( self, store, prefix, key, value, storage_class, bucket ): store.put(key, value) keyname = prefix + key if storage_class != 'STANDARD': pytest.xfail('boto does not support checking the storage class?') assert bucket.lookup(keyname).storage_class == storage_class def test_storage_class_putfile( self, store, prefix, key, value, storage_class, bucket ): store.put_file(key, BytesIO(value)) keyname = prefix + key if storage_class != 'STANDARD': pytest.xfail('boto does not support checking the storage class?') assert bucket.lookup(keyname).storage_class == storage_class class TestExtendedKeyspaceBotoStore(TestBotoStorage, ExtendedKeyspaceTests): @pytest.fixture def store(self, bucket, prefix, reduced_redundancy): class ExtendedKeyspaceStore(ExtendedKeyspaceMixin, BotoStore): pass return ExtendedKeyspaceStore(bucket, prefix, reduced_redundancy=reduced_redundancy) simplekv-0.14.1/tests/test_bucket_manager.py000066400000000000000000000011631364363245700211670ustar00rootroot00000000000000import pytest boto = pytest.importorskip('boto') from boto.exception import StorageResponseError from boto.s3.key import Key from bucket_manager import boto_credentials, boto_bucket @pytest.fixture(params=boto_credentials, ids=[c['access_key'] for c in boto_credentials]) def credentials(request): return request.param @pytest.yield_fixture() def bucket(credentials): with boto_bucket(**credentials) as bucket: yield bucket def test_simple(bucket): pass def test_simple_with_contents(bucket): k = bucket.new_key('i_will_prevent_deletion') k.set_contents_from_string('meh') simplekv-0.14.1/tests/test_cache.py000066400000000000000000000013231364363245700172610ustar00rootroot00000000000000from simplekv.memory import DictStore from simplekv.cache import CacheDecorator from basic_store import BasicStore import pytest class TestCache(BasicStore): # FIXME: this could use some extra combinations tested @pytest.fixture def front_store(self): return DictStore() @pytest.fixture def backing_store(self): return DictStore() @pytest.fixture def store(self, front_store, backing_store): return CacheDecorator(front_store, backing_store) def test_works_when_cache_loses_key(self, store, front_store, key, value): store.put(key, value) assert store.get(key) == value front_store.delete(key) assert store.get(key) == value simplekv-0.14.1/tests/test_dict_store.py000066400000000000000000000012231364363245700203540ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 from simplekv.memory import DictStore from basic_store import BasicStore from idgens import UUIDGen, HashGen from test_hmac import HMACDec import pytest from conftest import ExtendedKeyspaceTests from simplekv.contrib import ExtendedKeyspaceMixin class TestDictStore(BasicStore, UUIDGen, HashGen, HMACDec): @pytest.fixture def store(self): return DictStore() class TestExtendedKeyspaceDictStore(TestDictStore, ExtendedKeyspaceTests): @pytest.fixture def store(self): class ExtendedKeyspaceStore(ExtendedKeyspaceMixin, DictStore): pass return ExtendedKeyspaceStore() simplekv-0.14.1/tests/test_filesystem_store.py000066400000000000000000000145111364363245700216210ustar00rootroot00000000000000# coding: utf8 import os import stat from simplekv._compat import BytesIO, url_quote, url_unquote, PY2 import tempfile from simplekv._compat import urlparse from simplekv.fs import FilesystemStore, WebFilesystemStore from basic_store import BasicStore from url_store import UrlStore from idgens import UUIDGen, HashGen from conftest import ExtendedKeyspaceTests from simplekv.contrib import ExtendedKeyspaceMixin from mock import Mock import pytest class TestBaseFilesystemStore(BasicStore, UrlStore, UUIDGen, HashGen): @pytest.yield_fixture def tmpdir(self, tmp_path): yield str(tmp_path) @pytest.fixture def store(self, tmpdir): return FilesystemStore(tmpdir) class TestFilesystemStoreMkdir(TestBaseFilesystemStore): def test_concurrent_mkdir(self, tmpdir, mocker): # Concurrent instantiation of the store in two threads could lead to # the situation where both threads see that the directory does not # exists. For one, the call to mkdir succeeds, for the other it fails. # This is ok for us as long as the directory exists afterwards. makedirs = mocker.patch('os.makedirs') makedirs.side_effect = OSError("Failure") mocker.patch('os.path.isdir') store = FilesystemStore(os.path.join(tmpdir, 'test')) # We have mocked os.makedirs, so this won't work. But it should # pass beyond the OS error and simply fail on writing the file itself. if PY2: with pytest.raises(IOError): store.put('test', b'test') else: with pytest.raises(FileNotFoundError): store.put('test', b'test') class TestFilesystemStoreFileURI(TestBaseFilesystemStore): @pytest.mark.skipif(os.name != 'posix', reason='Not supported outside posix.') def test_correct_file_uri(self, store, tmpdir, key): expected = 'file://' + tmpdir + '/' + url_quote(key) assert store.url_for(key) == expected def test_file_uri(self, store, value): tmpfile = tempfile.NamedTemporaryFile(delete=False) try: tmpfile.write(value) tmpfile.close() key = store.put_file(u'testkey', tmpfile.name) url = store.url_for(key) assert url.startswith('file://') parts = urlparse(url) ndata = open(parts.path, 'rb').read() assert value == ndata finally: if os.path.exists(tmpfile.name): os.unlink(tmpfile.name) # runs each test with a nonstandard umask and checks if it is set correctly class TestFilesystemStoreUmask(TestBaseFilesystemStore): @pytest.fixture(scope='class') def current_umask(self): mask = os.umask(0) # re-set umask os.umask(mask) return mask @pytest.fixture(scope='class') def perms(self, current_umask): # the permissions we expect on files are inverse to the mask return 0o666 & (0o777 ^ current_umask) def test_file_permission_on_new_file_have_correct_value( self, store, perms, value ): src = BytesIO(value) key = store.put_file(u'test123', src) parts = urlparse(store.url_for(key)) path = parts.path mode = os.stat(path).st_mode mask = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert mode & mask == perms def test_file_permissions_on_moved_in_file_have_correct_value( self, store, perms, key, value ): tmpfile = tempfile.NamedTemporaryFile(delete=False) tmpfile.write(value) tmpfile.close() os.chmod(tmpfile.name, 0o777) try: key = store.put_file(key, tmpfile.name) parts = urlparse(store.url_for(key)) path = url_unquote(parts.path) mode = os.stat(path).st_mode mask = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert mode & mask == perms finally: if os.path.exists(tmpfile.name): os.path.unlink(tmpfile.name) class TestFileStoreSetPermissions(TestFilesystemStoreUmask): @pytest.fixture def perms(self): return 0o612 self.tmpdir = tempfile.mkdtemp() @pytest.fixture def store(self, tmpdir, perms): return FilesystemStore(tmpdir, perm=perms) class TestWebFileStore(TestBaseFilesystemStore): @pytest.fixture def url_prefix(self): return 'http://some/url/root/' @pytest.fixture def store(self, tmpdir, url_prefix): return WebFilesystemStore(tmpdir, url_prefix) def test_url(self, store, url_prefix, key): expected = url_prefix + url_quote(key) assert store.url_for(key) == expected def test_url_callable(self, tmpdir, key): prefix = 'http://some.prefix.invalid/' mock_callable = Mock(return_value=prefix) store = WebFilesystemStore(tmpdir, mock_callable) expected = prefix + url_quote(key) assert store.url_for(key) == expected mock_callable.assert_called_with(store, key) class TestExtendedKeyspaceFilesystemStore(TestBaseFilesystemStore, ExtendedKeyspaceTests): @pytest.fixture def store(self, tmpdir): class ExtendedKeyspaceStore(ExtendedKeyspaceMixin, FilesystemStore): pass return ExtendedKeyspaceStore(tmpdir) def test_prefix_iterator_ossep(self, store, value): delimiter = u"X" for k in [ u"a1" + os.sep + u"b1", u"a1" + os.sep + u"b1", u"a2" + os.sep + u"b1", u"a3", u"a4" + os.sep + u"b1" + os.sep + u"c1", u"a4" + os.sep + u"b1" + os.sep + u"c2", u"a4" + os.sep + u"b2" + os.sep + u"c1", u"a4" + os.sep + u"b3", ]: store.put(k, value) l = sorted(store.iter_prefixes(os.sep)) assert l == [ u"a1" + os.sep, u"a2" + os.sep, u"a3", u"a4" + os.sep, ] l = sorted(store.iter_prefixes( os.sep, prefix=u"a4" + os.sep, )) assert l == [ u"a4" + os.sep + "b1" + os.sep, u"a4" + os.sep + "b2" + os.sep, u"a4" + os.sep + "b3", ] l = sorted(store.iter_prefixes( os.sep, prefix=u"foo" + os.sep, )) assert l == [] simplekv-0.14.1/tests/test_git_store.py000066400000000000000000000045611364363245700202240ustar00rootroot00000000000000from basic_store import BasicStore from dulwich.repo import Repo from idgens import UUIDGen, HashGen import pytest from simplekv.git import GitCommitStore from conftest import ExtendedKeyspaceTests from simplekv.contrib import ExtendedKeyspaceMixin class TestGitCommitStore(BasicStore, UUIDGen, HashGen): @pytest.fixture(params=[b'master', b'not-master']) def branch(self, request): return request.param @pytest.fixture(params=[b'', b'/', b'subdir', b'sub/subdir/', b'/sub/subdir', b'/sub/subdir/']) def subdir_name(self, request): return request.param @pytest.yield_fixture def repo_path(self, tmp_path): Repo.init_bare(str(tmp_path)) yield str(tmp_path) @pytest.fixture def store(self, repo_path, branch, subdir_name): return GitCommitStore(repo_path, branch=branch, subdir=subdir_name) def test_uses_subdir(self, repo_path, store, subdir_name, branch): # add a key store.put(u'foo', b'bar') sdir = subdir_name.decode('ascii').strip('/') fn = 'foo' if sdir: fn = sdir + '/' + fn repo = Repo(repo_path) commit = repo[repo.refs[b'refs/heads/' + branch]] tree = repo[commit.tree] _, blob_id = tree.lookup_path(repo.__getitem__, fn.encode('ascii')) assert repo[blob_id].data == b'bar' # add a second key, resulting in a new commit and two keys available store.put(u'foo2', b'bar2') fn = 'foo' if sdir: fn = sdir + '/' + fn repo = Repo(repo_path) commit = repo[repo.refs[b'refs/heads/' + branch]] tree = repo[commit.tree] _, blob_id = tree.lookup_path(repo.__getitem__, fn.encode('ascii')) assert repo[blob_id].data == b'bar' fn2 = 'foo2' if sdir: fn2 = sdir + '/' + fn2 _, blob_id = tree.lookup_path(repo.__getitem__, fn2.encode('ascii')) assert repo[blob_id].data == b'bar2' class TestExtendedKeyspaceGitStore(TestGitCommitStore, ExtendedKeyspaceTests): @pytest.fixture def store(self, repo_path, branch, subdir_name): class ExtendedKeyspaceStore(ExtendedKeyspaceMixin, GitCommitStore): pass return ExtendedKeyspaceStore(repo_path, branch=branch, subdir=subdir_name) simplekv-0.14.1/tests/test_hmac.py000066400000000000000000000143201364363245700171270ustar00rootroot00000000000000import hmac import os from simplekv._compat import BytesIO, xrange import tempfile from simplekv.crypt import _HMACFileReader, VerificationException,\ HMACDecorator from six import b, indexbytes, int2byte import pytest class TestHMACFileReader(object): @pytest.fixture def bad_datas(self, value): val = value * 3 def _alter_byte(byte_string, pos): old_val = indexbytes(byte_string, pos) new_byte = int2byte((old_val + 1 % 255)) return byte_string[:pos] + new_byte + byte_string[pos + 1:] return (_alter_byte(val, i) for i in xrange(len(val))) @pytest.fixture def expected_digest(self, secret_key, value, hashfunc): return hmac.HMAC(secret_key, value, hashfunc).digest() @pytest.fixture def stored_blob(self, value, expected_digest): return value + expected_digest @pytest.fixture def create_reader(self, stored_blob, secret_key, hashfunc): return lambda: _HMACFileReader(hmac.HMAC(secret_key, None, hashfunc), BytesIO(stored_blob)) @pytest.fixture def chunk_sizes(self, value): return [10 ** n for n in xrange(2, 8)] def test_close(self, create_reader): reader = create_reader() assert not reader.source.closed reader.close() assert reader.source.closed def test_close_via_context(self, create_reader): reader = create_reader() assert not reader.source.closed with reader as r: assert r is reader assert reader.source.closed def test_reading_limit_0(self, create_reader): reader = create_reader() data = reader.read(0) assert isinstance(data, bytes) assert len(data) == 0 data = reader.read(0) assert isinstance(data, bytes) assert len(data) == 0 def test_reading_with_limit(self, secret_key, hashfunc, value, create_reader, chunk_sizes): # try for different read lengths for n in chunk_sizes: chunks = [] reader = create_reader() while True: r = reader.read(n) if not r: break chunks.append(r) assert b('').join(chunks) == value def test_manipulated_input_full_read( self, secret_key, value, bad_datas, hashfunc ): for bad_data in bad_datas: reader = _HMACFileReader( hmac.HMAC(secret_key, None, hashfunc), BytesIO(bad_data) ) with pytest.raises(VerificationException): reader.read() def test_manipulated_input_incremental_read( self, secret_key, bad_datas, hashfunc ): for bad_data in bad_datas: reader = _HMACFileReader( hmac.HMAC(secret_key, None, hashfunc), BytesIO(bad_data) ) with pytest.raises(VerificationException): bitesize = 100 while True: if len(reader.read(bitesize)) != bitesize: break def test_input_too_short(self, secret_key, hashfunc): with pytest.raises(VerificationException): _HMACFileReader( hmac.HMAC(secret_key, None, hashfunc), BytesIO(b('a')) ) def test_unbounded_read(self, value, create_reader): assert create_reader().read() == value # test the "real" HMACMixin: core functionality and checks # this only works with dicts, as we access the internal structures to # manipulate values class HMACDec(object): @pytest.fixture def hmacstore(self, secret_key, store): return HMACDecorator(secret_key, store) def test_get_fails_on_manipulation(self, hmacstore, key, value): hmacstore.put(key, value) hmacstore.d[key] += b('a') with pytest.raises(VerificationException): hmacstore.get(key) def test_copy_raises_not_implemented(self, store): with pytest.raises(NotImplementedError): HMACDecorator(b'secret', store).copy(u'src', u'dest') def test_put_file_obj(self, key, value, hmacstore): hmacstore.put_file(key, BytesIO(value)) assert hmacstore.get(key) == value def test_put_file_str(self, key, value, hmacstore): with tempfile.NamedTemporaryFile(mode='wb', delete=False) as f: f.write(value) hmacstore.put_file(key, f.name) assert hmacstore.get(key) == value def test_get_file_obj(self, key, value, hmacstore): hmacstore.put(key, value) b = BytesIO() hmacstore.get_file(key, b) assert b.getvalue() == value def test_get_file_non_writable_target(self, key, value, hmacstore): hmacstore.put(key, value) path = '/tmp/this/file/does/not/exist' with pytest.raises(IOError, match='Error opening {} for writing'.format(path)): hmacstore.get_file(key, path) def test_get_file_fails_on_manipulation(self, hmacstore, key, value): hmacstore.put(key, value) hmacstore.d[key] += b('a') with tempfile.TemporaryFile() as tmp: with pytest.raises(VerificationException): hmacstore.get_file(key, tmp) with tempfile.NamedTemporaryFile(delete=False) as tmp: try: with pytest.raises(VerificationException): hmacstore.get_file(key, tmp.name) finally: os.unlink(tmp.name) def test_open_fails_on_manipulation(self, hmacstore, key, value): hmacstore.put(key, value) hmacstore.d[key] += b('a') with pytest.raises(VerificationException): hmacstore.open(key).read() handle = hmacstore.open(key) # we read 1 extra byte now, because the value is actually longer handle.read(len(value) + 1) with pytest.raises(VerificationException): handle.read(1) def test_get_fails_on_replay_manipulation( self, hmacstore, key, key2, value ): hmacstore.put(key, value) hmacstore.d[key2] = hmacstore.d[key] hmacstore.get(key) with pytest.raises(VerificationException): hmacstore.get(key2) simplekv-0.14.1/tests/test_mongo.py000066400000000000000000000022711364363245700173400ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 from uuid import uuid4 as uuid import pytest pymongo = pytest.importorskip('pymongo') from simplekv.db.mongo import MongoStore from basic_store import BasicStore from conftest import ExtendedKeyspaceTests from simplekv.contrib import ExtendedKeyspaceMixin class TestMongoDB(BasicStore): @pytest.fixture def db_name(self): return '_simplekv_test_{}'.format(uuid()) @pytest.yield_fixture def store(self, db_name): try: conn = pymongo.MongoClient() except pymongo.errors.ConnectionFailure: pytest.skip('could not connect to mongodb') yield MongoStore(conn[db_name], 'simplekv-tests') conn.drop_database(db_name) class TestExtendedKeyspaceDictStore(TestMongoDB, ExtendedKeyspaceTests): @pytest.fixture def store(self, db_name): class ExtendedKeyspaceStore(ExtendedKeyspaceMixin, MongoStore): pass try: conn = pymongo.MongoClient() except pymongo.errors.ConnectionFailure: pytest.skip('could not connect to mongodb') yield ExtendedKeyspaceStore(conn[db_name], 'simplekv-tests') conn.drop_database(db_name) simplekv-0.14.1/tests/test_prefix_decorator.py000066400000000000000000000046141364363245700215630ustar00rootroot00000000000000from simplekv._compat import BytesIO from simplekv.memory import DictStore from simplekv.decorator import PrefixDecorator import pytest from basic_store import BasicStore class TestPrefixDecorator(BasicStore): @pytest.fixture(params=[ 'short_', 'loooooooooooooooooooooooooooooooooooooooooooooooooooooooooong_', 'nounderscore', '_129073ashd812g', ]) def prefix(self, request): return request.param @pytest.fixture(params=['prefix2_', 'zz', ]) def prefix2(self, request): # these are used when multiple prefixes in a single store # are requested return request.param @pytest.fixture(params=[True, False]) def store(self, request, prefix): base_store = DictStore() # do we add extra keys to the underlying store? if request.param: base_store.put(u'some_other_value', b'data1') base_store.put(u'ends_with_short_', b'data2') base_store.put(u'xx', b'data3') base_store.put(u'test', b'data4') return PrefixDecorator(prefix, base_store) def test_put_returns_correct_key(self, store, prefix, key, value): assert key == store.put(key, value) def test_put_sets_prefix(self, store, prefix, key, value): full_key = prefix + key key == store.put(key, value) assert store._dstore.get(full_key) == value def test_put_file_returns_correct_key(self, store, prefix, key, value): assert key == store.put_file(key, BytesIO(value)) def test_put_file_sets_prefix(self, store, prefix, key, value): full_key = prefix + key key == store.put_file(key, BytesIO(value)) assert store._dstore.get(full_key) == value def test_multiple_prefixes_one_store(self, store, prefix, prefix2, key, value): base_store = store._dstore store2 = PrefixDecorator(prefix2, base_store) pv = value + prefix.encode('ascii') pv2 = value + prefix2.encode('ascii') # put in with each prefix store.put(key, pv) store2.put(key, pv2) assert key in store assert key in store2 assert prefix + key in base_store assert prefix2 + key in base_store assert len(store.keys()) == 1 assert len(store2.keys()) == 1 assert store.get(key) == pv assert store2.get(key) == pv2 simplekv-0.14.1/tests/test_readonly_decorator.py000066400000000000000000000011501364363245700220730ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 from simplekv.decorator import ReadOnlyDecorator from simplekv.memory import DictStore import pytest class TestReadOnlyDecorator(object): def test_readonly(self): store0 = DictStore() store0.put(u"file1", b"content") store = ReadOnlyDecorator(store0) with pytest.raises(AttributeError): store.put(u"file1", b"content2") with pytest.raises(AttributeError): store.delete(u"file1") assert store.get(u"file1") == b"content" assert u"file1" in store assert set(store.keys()) == {u"file1"} simplekv-0.14.1/tests/test_redis.py000066400000000000000000000022411364363245700173240ustar00rootroot00000000000000#!/usr/bin/env python from basic_store import BasicStore, TTLStore from conftest import ExtendedKeyspaceTests from simplekv.contrib import ExtendedKeyspaceMixin import pytest redis = pytest.importorskip('redis') from redis import StrictRedis from redis.exceptions import ConnectionError class TestRedisStore(TTLStore, BasicStore): @pytest.yield_fixture() def store(self): from simplekv.memory.redisstore import RedisStore r = StrictRedis() try: r.get('anything') except ConnectionError: pytest.skip('Could not connect to redis server') r.flushdb() yield RedisStore(r) r.flushdb() class TestExtendedKeyspaceDictStore(TestRedisStore, ExtendedKeyspaceTests): @pytest.fixture def store(self): from simplekv.memory.redisstore import RedisStore class ExtendedKeyspaceStore(ExtendedKeyspaceMixin, RedisStore): pass r = StrictRedis() try: r.get('anything') except ConnectionError: pytest.skip('Could not connect to redis server') r.flushdb() yield ExtendedKeyspaceStore(r) r.flushdb() simplekv-0.14.1/tests/test_sqlalchemy.py000066400000000000000000000036131364363245700203640ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 import pytest sqlalchemy = pytest.importorskip('sqlalchemy') from sqlalchemy import create_engine, MetaData from sqlalchemy.exc import OperationalError from sqlalchemy.pool import StaticPool from simplekv.db.sql import SQLAlchemyStore from basic_store import BasicStore from conftest import ExtendedKeyspaceTests from simplekv.contrib import ExtendedKeyspaceMixin # FIXME: for local testing, this needs configurable dsns class TestSQLAlchemyStore(BasicStore): DSNS = [ ('pymysql', 'mysql+pymysql://travis:@localhost/simplekv_test'), ('psycopg2', 'postgresql+psycopg2://postgres:@127.0.0.1/simplekv_test'), ('sqlite3', 'sqlite:///:memory:') ] @pytest.fixture(params=DSNS, ids=[v[0] for v in DSNS]) def engine(self, request): module_name, dsn = request.param # check module is available pytest.importorskip(module_name) engine = create_engine(dsn, poolclass=StaticPool) try: engine.connect() except OperationalError: pytest.skip('could not connect to database {}'.format(dsn)) return engine @pytest.yield_fixture def store(self, engine): metadata = MetaData(bind=engine) store = SQLAlchemyStore(engine, metadata, 'simplekv_test') # create table store.table.create() yield store metadata.drop_all() class TestExtendedKeyspaceSQLAlchemyStore(TestSQLAlchemyStore, ExtendedKeyspaceTests): @pytest.fixture def store(self, engine): class ExtendedKeyspaceStore(ExtendedKeyspaceMixin, SQLAlchemyStore): pass metadata = MetaData(bind=engine) store = ExtendedKeyspaceStore(engine, metadata, 'simplekv_test') # create table store.table.create() yield store metadata.drop_all() simplekv-0.14.1/tests/test_url_encode_keys_decorator.py000066400000000000000000000020361364363245700234340ustar00rootroot00000000000000#!/usr/bin/env python # coding=utf8 from simplekv.decorator import URLEncodeKeysDecorator from simplekv.memory import DictStore import pytest from basic_store import BasicStore class TestURLEncodeKeysDecorator(BasicStore): @pytest.fixture() def base_store(self): return DictStore() @pytest.fixture() def store(self, base_store): return URLEncodeKeysDecorator(base_store) def test_urlencode(self, store): store.put(u'key special:-🍺', b'val1') assert store.get(u'key special:-🍺') == b'val1' def test_urlencode_base_store(self, store, base_store, value): store.put(u'abc def/key', value) assert u'abc+def%2Fkey' in base_store assert base_store.get(u'abc+def%2Fkey') == value assert store.get(u'abc def/key') == value # The invalid key is replaced by a valid one after encoding through # the decorator... test_exception_on_invalid_key_delete = None test_exception_on_invalid_key_get_file = None test_exception_on_invalid_key_get = None simplekv-0.14.1/tests/url_store.py000066400000000000000000000006761364363245700172070ustar00rootroot00000000000000# coding: utf8 import pytest class UrlStore(object): def test_url_for_for_generates_url_for(self, store, key, value): store.put(key, value) assert store.url_for(key) def test_url_for_generation_does_not_check_exists(self, store, key): store.url_for(key) def test_url_for_generation_checks_valid_key(self, store, invalid_key): with pytest.raises(ValueError): store.url_for(invalid_key) simplekv-0.14.1/tox.ini000066400000000000000000000020761364363245700147640ustar00rootroot00000000000000[tox] envlist = py{27,35,36,37,38},py36-old-azure-storage-blob [testenv] # See https://github.com/boto/boto/issues/3717 setenv= BOTO_CONFIG=/dev/null BOTO_PATH=/dev/null deps= # as workaround for https://github.com/pytest-dev/pytest/issues/6925 # use a somewhat older version of pytest: pytest <= 5.3.5 pycodestyle pytest-cov pytest-mock pytest-xdist mock redis psycopg2 sqlalchemy pymysql pymongo dulwich boto azure-storage-blob futures # ideally we would not need futures here but it doesn't work otherwise commands= pycodestyle --ignore=E402,E741 --max-line-length 98 simplekv tests pytest -n 4 --dist=loadfile --cov=simplekv -rs --doctest-modules simplekv/idgen.py simplekv/fs.py tests [testenv:py36-old-azure-storage-blob] # Test on one python version with the "old" code for azure-storage-blob<12 deps= pytest pytest-cov pytest-mock mock azure-storage-blob<12 # For some reason, switching on usedevelop makes collecting coverage data much more reliable usedevelop=True commands=pytest --cov=simplekv tests/test_azure_store.py