python利用文件锁,同时只允许一个实例运行

使用文件锁,支持linux和windows

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
import os
import sys
import time
import logging

logger = logging

class SingleInstance:
def __init__(self):
py_file_path = os.path.abspath(__file__)
basepath = os.path.dirname(py_file_path)
self.lockfile = os.path.normpath(basepath + '/' + os.path.basename(__file__) + '.lock')
if sys.platform == 'win32':
try:
# file already exists, we try to remove (in case previous execution was interrupted)
if os.path.exists(self.lockfile):
os.unlink(self.lockfile)

self.fd = os.open(self.lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR)
except OSError as e:
if e.errno == 13:
logger.error("Another instance is already running, quit.")
sys.exit(-1)
raise e
else:
# non Windows
import fcntl

self.fp = open(self.lockfile, 'w')
try:
fcntl.lockf(self.fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
logger.error("Another instance is already running, quit.")
sys.exit(-1)

if __name__ == '__main__':
instance = SingleInstance()
time.sleep(10)

hbase操作

建表

1
2
3
4
5
6
7
8
9
10
11
12
13
create 'table1', {
NAME => 'card',
DATA_BLOCK_ENCODING => 'NONE',
BLOOMFILTER => 'ROWCOL',
REPLICATION_SCOPE => '1',
COMPRESSION => 'SNAPPY',
VERSIONS => '1',
MIN_VERSIONS => '0',
KEEP_DELETED_CELLS => 'false',
BLOCKSIZE => '65536',
IN_MEMORY => 'false',
BLOCKCACHE => 'false'
}

数据操作

1
2
3
put 'table1', 'rowkey1', 'card:col3', 'value3'
get 'table1', 'rowkey1', {COLUMN => ['card:col1', 'card:col2']}
scan 'table1'

java

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
public void test() {
logger.info("test");

Configuration conf = HBaseConfiguration.create();
conf.set("hbase.master", "172.17.161.5:60000");
conf.set("hbase.zookeeper.property.clientport", "2181");
conf.set("hbase.zookeeper.quorum", "172.17.161.5");
conf.set("zookeeper.znode.parent", "/hbase-rpt");

try {
Connection connection = ConnectionFactory.createConnection(conf);
Admin admin = connection.getAdmin();

TableName[] names = admin.listTableNames();
for (TableName tableName : names) {
logger.info("Table Name is : " + tableName.getNameAsString());
}
} catch (IOException e) {
e.printStackTrace();
}
}

hdfs设置账号权限

设置目录的默认权限

1
2
3
4
# 设置目录的默认权限,这样用户新创建的目录和子目录就会自动有权限
hdfs dfs -setfacl -R -m default:group:group1:rwx /hive-dw/db1
# 设置已有目录的权限
hdfs dfs -setfacl -R -m group:group1:rwx /hive-dw/db1

centos开启ipv6

编辑/etc/sysconfig/network-scripts/ifcfg-eth0,添加

1
2
3
4
5
6
7
IPV6INIT=yes
IPV6ADDR=primary_ipv6_address/64
IPV6_DEFAULTGW=ipv6_gateway
IPV6_AUTOCONF=no
DNS1=2001:4860:4860::8844
DNS2=2001:4860:4860::8888
DNS3=209.244.0.3

测试

1
ping6 2001:4860:4860::8888

参考

1
https://www.digitalocean.com/community/tutorials/how-to-enable-ipv6-for-digitalocean-droplets

nodejs npm包安装

设置下载镜像地址

1
2
3
4
5
6
7
# 国内地址
npm config set registry https://registry.cnpmjs.org
# 淘宝npm代理
npm config set registry https://registry.npm.taobao.org
# 安装包错误 npm ERR! Unexpected end of JSON input while parsing near...
# 则切换回国内代理或者清除缓存
npm cache clean --force

代理

1
2
npm config set proxy http://127.0.0.1:1080
npm config delete proxy

windows

1
2
3
4
# 安装包时提示 MSBUILD : error MSB3428: 未能加载 Visual C++ 组件“VCBuild.exe”
# 则需要以管理员权限运行
npm install --global windows-build-tools
npm install -g node-gyp

安装sass

binding.node 下载地址:https://github.com/sass/node-sass/releases/
安装方法:

elasticsearch常用操作

使用aggregations,排序必须要开启fielddata

1
2
3
4
5
6
7
8
9
PUT my_index/_mapping/my_type
{
"properties": {
"my_field": {
"type": "text",
"fielddata": true
}
}
}

安装,参数设置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 内核设置
vm.swappiness=1
vm.max_map_count=262144
# 内存设置
export ES_HEAP_SIZE=8g
# 配置文件
path.data: /home/stat/elasticsearch-6.1.1/data
path.logs: /home/stat/elasticsearch-6.1.1/logs
indices.fielddata.cache.size: 20%
# 外部访问
network.host: 0.0.0.0
# [1]: max file descriptors [4096] for elasticsearch process is too low, increase to at least [65536]
# [2]: max number of threads [1024] for user [stat] is too low, increase to at least [4096]
# [3]: system call filters failed to install; check the logs and fix your configuration or disable system call filters at your own risk
修改 /etc/security/limits.conf
* soft nofile 65536
* hard nofile 131072
* soft nproc 4096
* hard nproc 4096
修改 /etc/security/limits.d/90-nproc.conf
* soft nproc 4096
要在Memory下面修改增加:
bootstrap.memory_lock: false
bootstrap.system_call_filter: false

mongodb常用操作

备份,恢复

1
2
3
4
5
6
mongodump -h127.0.0.1:27017 -o ./dump
mongorestore -h 127.0.0.1:27017 ./dump

# 可以导出json或csv
mongoexport -h 127.0.0.1 --db tb_page --collection tb_page --type json -o tb_page.json
mongoimport -h 127.0.0.1 --db tb_page --collection tb_page --type json --mode upsert --file tb_page.json

更新

1
db.users.update({"_id": "id1"}, {"$set": {"groups": []}})

插入

1
db.products.insert( { item: "card", qty: 15 } )

plotly使用渲染图标chart

使用bottle

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
import json

from bottle import route, run, template, Bottle, response

import argparse
import pandas as pd

from plotly import utils
import plotly.graph_objs as go

from requests.compat import json as _json

from influxdb import DataFrameClient

app = Bottle()

@app.hook('after_request')
def enable_cors():
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS'
response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token'

@app.route('/data')
def index():
client = DataFrameClient('127.0.0.1', '8086', database='dc')
res = client.query('select sum(succ), sum(fail) from p group by host limit 10')

data = []
for metric, df in res.items():
print df
trace = go.Bar(
x=df[df.columns[0]],
y=df[df.columns[1]],
name=str(metric),
)
data.append(trace)
layout = go.Layout()
data_json = _json.dumps(data, cls=utils.PlotlyJSONEncoder)
layout_json = _json.dumps(layout, cls=utils.PlotlyJSONEncoder)
data_dict = json.loads(data_json)
layout_dict = json.loads(layout_json)
return {'records': data_dict, 'layout': layout_dict}

@app.route('/pie')
def pie():
client = DataFrameClient('127.0.0.1', '8086', database='dc')
res = client.query('select sum(succ), sum(fail) from p group by host limit 10')

data = []
labels = []
pds = []
for metric, df in res.items():
print metric, df
print '---------', df.columns, df.columns.values
labels.append(str(metric) + df.columns[0])
labels.append(str(metric) + df.columns[1])
pds.append(df[df.columns[0]])
pds.append(df[df.columns[1]])
trace = go.Pie(
labels=labels,
values=pd.concat(pds),
name=str(metric),
)
data.append(trace)
layout = go.Layout()
data_json = _json.dumps(data, cls=utils.PlotlyJSONEncoder)
layout_json = _json.dumps(layout, cls=utils.PlotlyJSONEncoder)
data_dict = json.loads(data_json)
layout_dict = json.loads(layout_json)
return {'records': data_dict, 'layout': layout_dict}

@app.route('/line')
def line():
client = DataFrameClient('127.0.0.1', '8086', database='dc')
res = client.query('select sum(succ), sum(fail) from p where time > now() - 1d and succ > 0 group by time(1h) limit 10')

data = []
for metric, df in res.items():
print metric, df
print '---------', df.index
for col in df.columns.values:
trace = go.Scatter(
x=df.index,
y=df[col],
name=str(metric),
)
data.append(trace)
layout = go.Layout()
data_json = _json.dumps(data, cls=utils.PlotlyJSONEncoder)
layout_json = _json.dumps(layout, cls=utils.PlotlyJSONEncoder)
data_dict = json.loads(data_json)
layout_dict = json.loads(layout_json)
return {'records': data_dict, 'layout': layout_dict}

run(app, host='localhost', port=8080, debug=True)

html

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
<html>
<head>
<script src="http://code.jquery.com/jquery-3.2.1.min.js"></script>
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
</head>

<div id="plot" style="width:90%;height:600px;"></div>
<div id="plot_query"></div>

<script>
function makeplot() {
$.ajax({
url: 'http://127.0.0.1:8080/pie',
data: {
},
dataType: 'json',
success: function (result) {
console.log(result);
//result = JSON.parse(result);
console.log(result.records);
console.log(result.layout);
makePlotly(result.query, result.records, result.layout);
document.getElementById("plot_query").innerHTML = result.query;
}
});
};

function makePlotly(title, traces, layout){
var plotDiv = document.getElementById("plot");
Plotly.newPlot('plot', traces, layout);
};

makeplot();
</script>
</html>

django解决ajax提交时的csrf跨域问题

在js中添加代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
$(document).ready(function() {
// using jQuery
function getCookie(name) {
var cookieValue = null;
if (document.cookie && document.cookie !== '') {
var cookies = document.cookie.split(';');
for (var i = 0; i < cookies.length; i++) {
var cookie = jQuery.trim(cookies[i]);
// Does this cookie string begin with the name we want?
if (cookie.substring(0, name.length + 1) === (name + '=')) {
cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
break;
}
}
}
return cookieValue;
}
var csrftoken = getCookie('csrftoken');

function csrfSafeMethod(method) {
// these HTTP methods do not require CSRF protection
return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method));
}
$.ajaxSetup({
beforeSend: function(xhr, settings) {
if (!csrfSafeMethod(settings.type) && !this.crossDomain) {
xhr.setRequestHeader("X-CSRFToken", csrftoken);
}
}
});
});