Skip to content

周伟奇 / simple_config

Go to a project
Toggle navigation
Toggle navigation pinning
e282cfeb by Gruel
Options

simple config

0 parents
Inline Side-by-side
Showing 15 changed files with 872 additions and 0 deletions
.gitignore 0 → 100755
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pycharm file
.idea/
.setup.cfg 0 → 100755
[nosetests]
verbosity=3
with-doctest=1
with-coverage=1
.style.yapf 0 → 100755
[style]
based_on_style = pep8
spaces_before_comment = 4
split_before_logical_operator = true
coalesce_brackets = true
ALLOW_SPLIT_BEFORE_DICT_VALUE=false
README.md 0 → 100755
# simple_config
Python项目的配置信息工具，可以从解析配置文件、环境变量。
## Getting Started
### Installing
```
$: git clone git@gitlab.situdata.com:zhouweiqi/simple_config.git
$: pip install -e simple_config
```
### QuickStart
**配置文件**
创建配置文件`test.ini`, 内容如下：
```
ETCD_HOST=1.1.1.1
ETCD_PORT=80
ETCD_PASSWD=0hyxdryq_CZ
ETCD_USER=test
HTTP_SERVER=http://test.com
DEBUG=true
ALLOWED_HOSTS=*.test.com, api.test.com, *
USER_NAME=root
```
解析配置文件
```python
In [2]: from simple_config import Config, ConfigAttribute, converter
In [3]: class ProjectConfig(Config):
...: ETCD_HOST = ConfigAttribute('ETCD_HOST')
...: ETCD_PORT = ConfigAttribute('ETCD_PORT', get_converter=int)
...: ETCD_USER = ConfigAttribute('ETCD_USER')
...: ETCD_PASSWD = ConfigAttribute('ETCD_PASSWD')
...:
...: HTTP_SERVER = ConfigAttribute(
...: 'HTTP_SERVER', get_converter=converter.server)
...:
...: DEBUG = ConfigAttribute('DEBUG', get_converter=converter.boolean)
...: ALIAS_DEBUG = ConfigAttribute('DEBUG', get_converter=converter.boolean)
...:
...: ALLOWED_HOSTS = ConfigAttribute(
...: 'ALLOWED_HOSTS', get_converter=converter.Csv())
...:
...:
...: config = ProjectConfig(defaults={'DEBUG': False})
In [4]: config.DEBUG
Out[4]: False
In [5]: config.from_env_file('test.ini')
In [6]: config.DEBUG
Out[6]: True
In [7]: config.ETCD_PORT
Out[7]: 80
In [8]: config.get_namespace('ETCD_')
Out[8]: {'host': '1.1.1.1', 'passwd': '0hyxdryq_CZ', 'port': 80, 'user': 'test'}
In [9]: config.ALLOWED_HOSTS
Out[9]: ['*.test.com', 'api.test.com', '*']
In [10]: config.ALIAS_DEBUG
Out[10]: True
In [11]: config.USER_NAME # 获取配置文件中有，但是Config没有声明的配置
Out[11]: 'root'
In [12]: config.HTTP_SERVER
Out[12]: Server(scheme='http', host='test.com', port=80)
```
**常量**
创建文件`const.py`, 内容如下
```python
import sys
from simple_config import Const
A = 1
B = 2
sys.modules[__name__] = Const.from_current_module(__name__)
```
加载常量
```python
In [1]: import const
In [2]: const
Out[2]: <simple_config.Const at 0x7fc59c9475c0>
In [3]: const?
Type: Const
String form: <simple_config.Const object at 0x7fc59c9475c0>
Docstring: <no docstring>
In [4]: const.A
Out[4]: 1
In [5]: const.B
Out[5]: 2
In [6]: const.A = 3 # 不能修改已有变量值
---------------------------------------------------------------------------
ConstError Traceback (most recent call last)
<ipython-input-6-d1e1d5d0fa5a> in <module>
----> 1 const.A = 3
~/code/simple_config/simple_config/__init__.py in __setattr__(self, name, value)
14 def __setattr__(self, name, value):
15 if name in self.__dict__:
---> 16 raise self.ConstError("Can't change const.{}".format(name))
17 if not name.isupper():
18 raise self.ConstCaseError(
ConstError: Can't change const.A
```
## Test
1. 安装依赖
```bash
$ pip install nose coverage
```
2. 运行测试
```
nosetests -c .setup.cfg
```
requirements/test_requirements.txt 0 → 100755
coverage==4.5.2
nose==1.3.7
setup.py 0 → 100755
import setuptools
import simple_config
with open('README.md', 'r', encoding='utf8') as fh:
long_description = fh.read()
setuptools.setup(
name="simple_config",
version=simple_config.__version__,
author="clay",
author_email="zhouweiqi@situdata.com",
description="A small tool to parse and store project config",
long_description=long_description,
long_description_content_type="text/markdown",
packages=setuptools.find_packages(),
classifiers=[
"Programming Language :: Python :: 3",
"Operating System :: OS Independent",
],
)
simple_config/__init__.py 0 → 100755
import sys
from .core import Config
from .core import Attribute as ConfigAttribute
class Const(object):
class ConstError(TypeError):
pass
class ConstCaseError(ConstError):
pass
def __setattr__(self, name, value):
if name in self.__dict__:
raise self.ConstError("Can't change const.{}".format(name))
if not name.isupper():
raise self.ConstCaseError(
"const name {} is not all uppercase".format(name))
self.__dict__[name] = value
@classmethod
def from_current_module(cls, name):
current_module = sys.modules[name]
_const = cls()
for key in dir(current_module):
if not key.isupper():
continue
setattr(_const, key, getattr(current_module, key))
return _const
__name__ = "simple_config"
__all__ = ('Config', 'ConfigAttribute', 'const')
__version__ = '1.2'
simple_config/converter.py 0 → 100755
from collections import namedtuple
import re
import string
from shlex import shlex
from .exceptions import ConvertError
__all__ = ('host_port', 'server', 'Csv')
Server = namedtuple('Server', 'scheme host port')
host_port_pattern = re.compile(
r'^(?P<host>[0-9a-zA-Z\-.]+)(?:\:(?P<port>\d{1,5}))?$')
def host_port(string):
"""
解析"host:port"字符串
>>> host_port('dev.some-host.org:60000')
Server(scheme=None, host='dev.some-host.org', port=60000)
>>> host_port('8.8.8.8:8080')
Server(scheme=None, host='8.8.8.8', port=8080)
>>> host_port('8.8.8.8')
Server(scheme=None, host='8.8.8.8', port=None)
>>> res = host_port('dev.some-host.org:600000')
Traceback (most recent call last):
...
simple_config.exceptions.ConvertError: invalid host_port \
string: dev.some-host.org:600000
"""
match = host_port_pattern.match(string)
if not match:
raise ConvertError('invalid host_port string: %s' % string)
port = match.group('port') and int(match.group('port'))
return Server(None, match.group('host'), port)
http_server_pattern = re.compile(
r'^(?P<schema>https?)://(?P<host>[0-9a-zA-Z\-.]+)(?:\:(?P<port>\d{1,5}))?$'
)
def server(string):
"""
解析"http://host:port"字符串
>>> server('http://dev.some-host.org:60000')
Server(scheme='http', host='dev.some-host.org', port=60000)
>>> server('https://8.8.8.8:8080')
Server(scheme='https', host='8.8.8.8', port=8080)
>>> server('http://8.8.8.8')
Server(scheme='http', host='8.8.8.8', port=80)
>>> res = server('dev.some-host.org:600000')
Traceback (most recent call last):
...
simple_config.exceptions.ConvertError: invalid http_server \
string: dev.some-host.org:600000
"""
match = http_server_pattern.match(string)
if not match:
raise ConvertError('invalid http_server string: %s' % string)
port = match.group('port') and int(match.group('port'))
if not port:
port = 80 if match.group('schema') == 'http' else 443
return Server(match.group('schema'), match.group('host'), port)
def boolean(instance):
"""
根据输入，返回是否为True
>>> boolean(True)
True
>>> boolean('True')
True
>>> boolean('TRUE')
True
>>> boolean('false')
False
>>> boolean(None)
Traceback (most recent call last):
...
simple_config.exceptions.ConvertError: expected boolean or str, \
but get <class 'type'>
"""
if isinstance(instance, bool):
return instance
elif isinstance(instance, str):
if instance.upper() == 'TRUE':
return True
return False
raise ConvertError('expected boolean or str, but get %s' % type(object))
class Csv(object):
"""
Produces a csv parser that return a list of transformed elements.
origin code is here
https://github.com/henriquebastos/python-decouple/blob/master/decouple.py
>>> csv = Csv()
>>> hosts = "*,*.test.com, demo.test.com ,test.org"
>>> csv(hosts)
['*', '*.test.com', 'demo.test.com', 'test.org']
>>> csv = Csv(delimiter=';；')
>>> string = 'a;b; c；D'
>>> csv(string)
['a', 'b', 'c', 'D']
>>> hosts = "*,*.test.com, demo.test.com ,test.org"
>>> csv(hosts)
['*,*.test.com, demo.test.com ,test.org']
"""
def __init__(self,
cast=str,
delimiter=',',
strip=string.whitespace,
post_process=list):
"""
Parameters:
cast -- callable that transforms the item just before
it's added to the list.
delimiter -- string of delimiters chars passed to shlex.
strip -- string of non-relevant characters to be passed to
str.strip after the split.
"""
self.cast = cast
self.delimiter = delimiter
self.strip = strip
self.post_process = post_process
def __call__(self, value):
"""The actual transformation"""
def transform(s):
return self.cast(s.strip(self.strip))
splitter = shlex(value, posix=True)
splitter.whitespace = self.delimiter
splitter.whitespace_split = True
return self.post_process(transform(s) for s in splitter)
simple_config/core.py 0 → 100755
from configparser import ConfigParser
import os
import re
import warnings
from .exceptions import (FileDoesNotExist, VariableNotExist, NotConfigured,
InvalidConfig)
class Attribute(object):
"""
>>> from .converter import boolean
>>> class TestConfig(Config):
... DEBUG = Attribute('DEBUG', boolean)
...
>>> config = TestConfig()
>>> config.DEBUG
Traceback (most recent call last):
....
simple_config.exceptions.NotConfigured: value of 'DEBUG' has not been set
>>> config = TestConfig(defaults={'debug':False})
>>> config.DEBUG
False
>>> config.DEBUG = False
>>> config._raw_config['debug']
Traceback (most recent call last):
....
KeyError: 'debug'
>>> config._raw_config['DEBUG']
False
"""
def __init__(self, name, get_converter=None):
if name.upper() != name:
warnings.warn('upper case name is expected')
self.__name__ = name.upper()
self.get_converter = get_converter
def __get__(self, obj, type=None):
if obj is None:
return self
try:
rv = obj._raw_config[self.__name__]
except KeyError:
raise NotConfigured(
"value of '%s' has not been set" % self.__name__)
if self.get_converter is not None:
rv = self.get_converter(rv)
return rv
def __set__(self, obj, value):
obj._raw_config[self.__name__] = value
class Environment(dict):
def __init__(self, *args, **kwargs):
super(Environment, self).__init__(*args, **kwargs)
@classmethod
def from_env_file(cls, filename, load_env=False):
"""从文件中加载环境变量
"""
if not os.path.exists(filename):
raise FileDoesNotExist('cannot find file: %s' % filename)
with open(filename, 'r', encoding='utf8') as f:
lines = f.readlines()
dct = dict(os.environ) if load_env else {}
for line in lines:
if line.startswith('#'):
continue
if '=' not in line:
if re.match(r'^\s*$', line):
continue
raise InvalidConfig("config line should contain '='")
key, value = line.strip().split('=', 1)
if value[0] == value[-1] and value[0] in ('\'', '"'):
warnings.warn('There is no need to quote string')
dct[key] = value
return cls(dct)
@classmethod
def from_env(cls):
return cls(dict(os.environ))
class Config(object):
INI_SECTION_NAME = 'settings'
def __init__(self, defaults=None):
self._raw_config = {}
defaults and self._update_config(**defaults)
def from_ini(self, filename):
"""
>>> config = Config(defaults={'DBEUG': True})
>>> from tempfile import TemporaryFile
>>> filename = TemporaryFile().name
>>> with open(filename, 'w', encoding='utf8') as f:
... f.write('[settings]\\ndebug=false\\n')
23
>>> config.from_ini(filename)
>>> config._raw_config['DEBUG']
'false'
>>> with open(filename, 'w', encoding='utf8') as f:
... f.write('[settings]\\ndebug=True\\n')
22
>>> config.from_ini(filename)
>>> config._raw_config['DEBUG']
'True'
>>> os.remove(filename)
"""
config = ConfigParser()
with open(filename, 'r', encoding='utf8') as f:
config.read_file(f)
self._update_config(**config[self.INI_SECTION_NAME])
def from_env_file(self, filename, load_env=False):
"""
>>> config = Config(defaults={'DBEUG': True})
>>> from tempfile import TemporaryFile
>>> filename = TemporaryFile().name
>>> with open(filename, 'w', encoding='utf8') as f:
... f.write('debug=false')
11
>>> config.from_env_file(filename)
>>> config._raw_config['DEBUG']
'false'
>>> with open(filename, 'w', encoding='utf8') as f:
... f.write('debug=True')
10
>>> config.from_env_file(filename)
>>> config._raw_config['DEBUG']
'True'
>>> os.remove(filename)
"""
self._update_config(**Environment.from_env_file(
filename, load_env=load_env))
def from_env_var(self, variable, errors='strict'):
environ = Environment.from_env()
value = environ.get(variable)
if value is None and errors == 'strict':
raise VariableNotExist('cannot get %s from env' % variable)
self._update_config(**{variable: value})
def from_object(self, obj):
for key in dir(obj):
if key.isupper():
self._update_config(**{key: getattr(obj, key)})
def from_dict(self, dct):
"""
>>> from .converter import boolean
>>> class NewConfig(Config):
... DEBUG = Attribute('DEBUG', get_converter=boolean)
...
>>> config = NewConfig()
>>> config.DEBUG
Traceback (most recent call last):
....
simple_config.exceptions.NotConfigured: \
value of 'DEBUG' has not been set
>>> config.from_dict({'debug': 'true'})
>>> config.DEBUG
True
>>> config.from_dict({'Debug': ''})
>>> config.DEBUG
False
"""
self._update_config(**dct)
def _update_config(self, **kwargs):
for key, value in kwargs.items():
self._raw_config[key.upper()] = value
def __setattr__(self, key, value):
"""
>>> config = Config()
>>> config._raw_config['DEBUG']
Traceback (most recent call last):
....
KeyError: 'DEBUG'
>>> config.DEBUG = False
>>> config._raw_config
{'DEBUG': False}
"""
pre_fields = ('_raw_config', )
if key in pre_fields:
super().__setattr__(key, value)
else:
if key.upper() != key:
warnings.warn('upper case attribute is expected')
key = key.upper()
self._update_config(**{key: value})
def get_namespace(self, namespace, lowercase=True, trim_namespace=True):
"""
>>> config = Config(
... defaults={'A_ID': 1, 'A_NAME': 2, 'A_CITY': 3, 'B_ID': 2})
>>> config.get_namespace('A') == {'_id': 1, '_name': 2, '_city': 3}
True
>>> config.get_namespace('A_') == {'id': 1, 'name': 2, 'city': 3}
True
>>> config.get_namespace('A_', lowercase=False) \\
... == {'ID': 1, 'NAME': 2, 'CITY': 3}
True
>>> config.get_namespace(
... 'A_', lowercase=False, trim_namespace=False) \\
... == {'A_ID': 1, 'A_NAME': 2, 'A_CITY': 3}
True
"""
dct = {}
for key in self._raw_config.keys():
origin_key = key
if not key.startswith(namespace):
continue
if trim_namespace:
key = key[len(namespace):]
# if lowercase:
# key = key.lower()
dct[key] = getattr(self, origin_key)
return dct
def __getattr__(self, key):
try:
return self._raw_config[key]
except KeyError:
raise AttributeError("'%s' object has no attribute '%s'" %
(self.__class__, key))
simple_config/exceptions.py 0 → 100755
class BaseError(Exception):
pass
class ConvertError(BaseError):
pass
class FileDoesNotExist(BaseError):
pass
class VariableNotExist(BaseError):
pass
class NotConfigured(BaseError):
pass
class InvalidConfig(BaseError):
pass
test/_const_demo.py 0 → 100755
import sys
from simple_config import Const
A = 1
A = 2
# b = 1
sys.modules[__name__] = Const.from_current_module(__name__)
test/test.ini 0 → 100755
[settings]
ETCD_HOST=1.1.1.1
ETCD_PORT=80
ETCD_PASSWD=0hyxdryq_CZ
ETCD_USER=test
HTTP_SERVER=http://test.com
DEBUG=true
ALLOWED_HOSTS=*.test.com, api.test.com, *
USER_NAME=root
root
root
root
test/test_config.py 0 → 100755
import os
from simple_config import Config, ConfigAttribute, converter
from simple_config import exceptions
file_dir = os.path.dirname(os.path.abspath(__file__))
class ProjectConfig(Config):
ETCD_HOST = ConfigAttribute('ETCD_HOST')
ETCD_PORT = ConfigAttribute('ETCD_PORT', get_converter=int)
ETCD_USER = ConfigAttribute('ETCD_USER')
ETCD_PASSWD = ConfigAttribute('ETCD_PASSWD')
HTTP_SERVER = ConfigAttribute(
'HTTP_SERVER', get_converter=converter.server)
DEBUG = ConfigAttribute('DEBUG', get_converter=converter.boolean)
ALIAS_DEBUG = ConfigAttribute('DEBUG', get_converter=converter.boolean)
ALLOWED_HOSTS = ConfigAttribute(
'ALLOWED_HOSTS', get_converter=converter.Csv())
def test_defaults():
config = ProjectConfig(defaults={'DEBUG': False, 'ETCD_HOST': 'test'})
assert config.DEBUG is False
assert config.ALIAS_DEBUG is False
assert config.ETCD_HOST == 'test'
try:
config.ETCD_PORT
except exceptions.NotConfigured:
pass
else:
raise Exception('exception is expected')
def test_env_file():
config = ProjectConfig(defaults={'DEBUG': False})
assert config.DEBUG is False
config.from_env_file(os.path.join(file_dir, 'test_env'))
assert config.DEBUG is True
assert config.ETCD_PORT == 80
assert config.get_namespace('ETCD_') == {
'host': '1.1.1.1',
'passwd': '0hyxdryq_CZ',
'port': 80,
'user': 'test'
}
assert config.ALLOWED_HOSTS == ['*.test.com', 'api.test.com', '*']
assert config.ALIAS_DEBUG is True
assert config.USER_NAME == 'root'
def test_ini():
config = ProjectConfig(defaults={'DEBUG': False})
assert config.DEBUG is False
config.from_ini(os.path.join(file_dir, 'test.ini'))
assert config.DEBUG is True
assert config.ETCD_PORT == 80
assert config.get_namespace('ETCD_') == {
'host': '1.1.1.1',
'passwd': '0hyxdryq_CZ',
'port': 80,
'user': 'test'
}
assert config.ALLOWED_HOSTS == ['*.test.com', 'api.test.com', '*']
assert config.ALIAS_DEBUG is True
assert config.USER_NAME == 'root\nroot\nroot\nroot'
test/test_const.py 0 → 100755
import importlib.machinery
import os
filename = os.path.join(
os.path.dirname(os.path.abspath(__file__)), '_const_demo.py')
loader = importlib.machinery.SourceFileLoader('const_demo', filename)
const_demo = loader.load_module()
def test_const():
# 变量只能被赋值一次
const_demo.B = 2
try:
const_demo.B = 3
except const_demo.ConstError:
pass
else:
raise Exception('ConstError is expected')
# module中赋值的变量能够被自动加载
assert const_demo.A == 2
try:
const_demo.A = 2
except const_demo.ConstError:
pass
else:
raise Exception('ConstError is expected')
assert not hasattr(const_demo, 'b')
test/test_env 0 → 100755
ETCD_HOST=1.1.1.1
ETCD_PORT=80
ETCD_PASSWD=0hyxdryq_CZ
ETCD_USER=test
HTTP_SERVER=http://test.com
DEBUG=true
ALLOWED_HOSTS=*.test.com, api.test.com, *
USER_NAME=root
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!