python 时间格式转换

datetime和str

1
2
3
4
5
6
# str转datetime
datetime.datetime.strptime('2017-04-23 00:00:00', '%Y-%m-%d %H:%M:%S')
# datetime转str
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S');
# datetime转timestamp
time.mktime(dt.timetuple())

utc时间转本地时间

1
2
3
4
5
6
from_zone = tz.tzutc()
to_zone = tz.tzlocal()
utc = datetime.strptime(utcs, '%Y-%m-%dT%H:%M:%SZ')
utc = utc.replace(tzinfo=from_zone)
local_dt = utc.astimezone(to_zone)
dts = local_dt.strftime('%Y-%m-%d %H:%M')

datetime转timestamp

1
2
datetime.fromtimestamp(ts)
datetime.utcfromtimestamp(ts)

python设置日志翻滚

可翻滚的日志: logger.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
# -*- coding: utf-8 -*-
import os
import logging
from logging.handlers import RotatingFileHandler

def get_full_path(local_path):
file_path = os.path.abspath(__file__)
base_path = os.path.dirname(file_path)
return os.path.join(base_path, local_path)

LOG_LEVEL = logging.DEBUG
LOG_FORMAT = '%(asctime)s %(levelname)s [%(filename)s:%(funcName)s:%(lineno)s] %(message)s'
DATEFMT = '%Y-%m-%d %H:%M:%S'
LOG_FILE_NAME = get_full_path('logs/app.log')
LOG_FILE_MAX_SIZE = 10 * 1024 * 1024
LOG_FILE_BACKUP_COUNT = 5

formatter = logging.Formatter(LOG_FORMAT, datefmt=DATEFMT)

console = logging.StreamHandler()
console.setFormatter(formatter)

rotate = RotatingFileHandler(LOG_FILE_NAME, maxBytes=LOG_FILE_MAX_SIZE, backupCount=LOG_FILE_BACKUP_COUNT)
rotate.setFormatter(formatter)

logger = logging.getLogger('app_common')
logger.propagate = False
logger.setLevel(LOG_LEVEL)
logger.addHandler(console)
logger.addHandler(rotate)

使用方法

1
2
3
from logger import logger

logger.info('log')

curl

post json

1
curl -H "Content-Type: application/json" -X POST -d '{"a":"b","c":10}' http://127.0.0.1 -w %{time_connect}:%{time_starttransfer}:%{time_total}

post form

1
curl -d "param1=value1&param2=value2" "http://127.0.0.1/test"

python http请求

http请求

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
def open_url(url, method='GET', headers=None, postform=None, postdata=None, timeout=10):
"""http请求"""
try:
if headers is None:
headers = {}
if postform is None:
postform = {}

if len(postform) > 0:
data = urllib.urlencode(postform)
headers['Content-Type'] = 'application/x-www-form-urlencoded'
else:
data = postdata

req = urllib2.Request(url=url, data=data, headers=headers)
req.get_method = lambda: method.upper()
res = urllib2.urlopen(req, timeout=timeout)
data = res.read()
res.close()
return res.code, res.headers, data
except urllib2.HTTPError, e:
data = e.read()
e.close()
return e.code, e.headers, data
except Exception:
log = traceback.format_exc().replace('\n', ' ')
return 500, {}, log

beeline连接

连接

1
beeline -n hive -p hive -u jdbc:hive2://127.0.0.1:10000

执行

1
2
3
4
5
6
7
/usr/bin/beeline --color=true --showHeader=true --outputformat=tsv2 -n hive -p hive -u jdbc:hive2://hiveserver2.simpledao.com:10000 -e "
set hive.merge.mapredfiles=true;
set hive.merge.mapfiles=true;
set hive.merge.smallfiles.avgsize=500000000;
set mapred.max.split.size=4000000000;

select * from table1 limit 10;"

flask wtf form template渲染

flask wtf form template渲染

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
<form action="" method="POST" role="form" class="admin-form form-horizontal" enctype="multipart/form-data">
{{ task_form.csrf_token }}

{{ task_form.hidden_tag() }}

{% for field, widget in task_form._fields.iteritems() %}
{% if field != "csrf_token" %}
<ul class="col-md-offset-2" style="color:red">
{{ ', '.join(task_form.errors[field]) }}
</ul>

<div class="form-group">
{{ task_form[field].label(class="col-md-2 control-label") }}

<div class="col-md-10">
{{ task_form[field](class="form-control") }}
</div>
</div>
{% endif %}
{% endfor %}

<div class="form-group">
<div class="col-md-offset-2 col-md-10 submit-row">
<input type="submit" class="btn btn-primary" value="保存">
</div>
</div>

</form>

kafka常用命令

查看topic列表

1
kafka-topics --zookeeper 127.0.0.1 --list

创建topic

1
kafka-topics.sh --create --zookeeper 127.0.0.1:2181/kafkastreaming --replication-factor 2 --partitions 24 --topic test

删除topic

1
kafka-topics.sh --delete --zookeeper 127.0.0.1:2181/kafkastreaming --topic test

修改分区数

1
kafka-topics --zookeeper 127.0.0.1  --alter --topic topic1 --partitions 16

生产

1
kafka-console-producer.sh --broker-list 127.0.0.1:9092 --topic test

消费

1
kafka-console-consumer.sh --bootstrap-server 127.0.0.1:9092 --topic test --from-beginning