notebook20180415
程序员文章站
2022-05-28 11:42:11
...
curl
curl -F "userid=1" -F "filecomment=This is an image file" -F "[email protected]/home/user1/Desktop/test.jpg" localhost/uploader.php
curl -i -F "[email protected]样例.xlsx;type=application/octet-stream" http://127.0.0.1:5001/upload_overdue_file
curl --request POST --data-binary "c:/样例.xlsx" --header "application/octet-stream" http://127.0.0.1:5001/upload_overdue_file
multipart/form-data
curl -i -X POST -H "Content-Type: multipart/form-data" -F "file=c:/样例.xlsx" http://127.0.0.1:5001/upload_overdue_file
curl -i -X POST -H "Content-Type: multipart/form-data" -F "file=c:/样例.xlsx" http://127.0.0.1:5001/upload_overdue_file
curl -F "file=c:/样例.xlsx" -F "yourname=Daniel" -F "filedescription=Cool text file with cool text inside" http://127.0.0.1:5001/upload_file
curl -v -H "Content-Type: application/x-www-form-urlencoded" -F "[email protected]:/样例.xlsx" http://127.0.0.1:5001/upload_file
#FINAL, CHANGE FILE permission
curl -v -F "[email protected]:/test.xlsx" http://127.0.0.1:5001/upload_filefree
free -g && sync && echo 3 > /proc/sys/vm/drop_caches && free -gUpgrade Mysql 5.6 to 5.7
https://dev.mysql.com/doc/refman/5.7/en/upgrading.html#upgrade-methods
1. backup
mysqldump -u root -p --add-drop-table --routines --events --all-databases --force > data-for-upgrade.sql
2. shutdown
mysql -u root -p --execute="SET GLOBAL innodb_fast_shutdown=0"
mysqladmin -u root -p shutdown
sudo service mysqld status
3. upgrade binary with yum
wget https://repo.mysql.com//mysql57-community-release-el7-11.noarch.rpm
sudo yum localinstall mysql57-community-release-el7-11.noarch.rpm
4. mysqld_safe --user=mysql --datadir=/mnt/mysql
5. mysql_upgrade -u root -p
wget https://repo.mysql.com//mysql57-community-release-el7-11.noarch.rpm
sudo yum localinstall mysql57-community-release-el7-11.noarch.rpm -y
sudo yum install mysql-community-client.x86_64 -y
sudo service mysqld startUrl decode on Mysql
https://blog.cles.jp/item/4564
http://psoug.org/snippet/mySQL-url_decode-that-respects-utf8-characters-_502.htm
DELIMITER $$
DROP FUNCTION IF EXISTS `url_decode` $$
CREATE DEFINER=`root`@`%` FUNCTION `url_decode`(original_text TEXT) RETURNS TEXT CHARSET utf8
BEGIN
DECLARE new_text TEXT DEFAULT NULL;
DECLARE pointer INT DEFAULT 1;
DECLARE end_pointer INT DEFAULT 1;
DECLARE encoded_text TEXT DEFAULT NULL;
DECLARE result_text TEXT DEFAULT NULL;
SET new_text = REPLACE(original_text,'+',' ');
SET pointer = LOCATE("%", new_text);
while pointer <> 0 && pointer < (CHAR_LENGTH(new_text) - 2) DO
SET end_pointer = pointer + 3;
while MID(new_text, end_pointer, 1) = "%" DO
SET end_pointer = end_pointer+3;
END while;
SET encoded_text = MID(new_text, pointer, end_pointer - pointer);
SET result_text = CONVERT(UNHEX(REPLACE(encoded_text, "%", "")) USING utf8);
SET new_text = REPLACE(new_text, encoded_text, result_text);
SET pointer = LOCATE("%", new_text, pointer + CHAR_LENGTH(result_text));
END while;
return new_text;
END $$
DELIMITER ;Nginx+Gunicorn+Flask
1. sudo apt-get install nginx
# site config
/etc/nginx/sites-available/mysite
--------
upstream my_servers {
server unix:/home/hello/gunicorn.sock fail_timeout=0;
}
server {
server_tokens off;
listen 5005 ssl;
ssl on;
ssl_certificate /home/hello/ca/server.crt;
ssl_certificate_key /home/hello/ca/server.key;
ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
access_log /var/log/nginx/my.access.log;
gzip on;
gzip_types *;
gzip_proxied any;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://my_servers;
}
}
http {
log_format upstream_time '$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"'
'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
server {
access_log /spool/logs/nginx-access.log upstream_time;
...
}
}
--------
sudo ln -s /etc/nginx/sites-available/mysite /etc/nginx/sites-enabled/mysite
sudo /etc/init.d/nginx restart
2. Gunicorn
gunicorn -b unix:/home/hello/gunicorn.sock --name appname -w 4 --max-requests 1000 wsgi:appCentOS 7 Nginx
sudo rpm -Uvh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm
sudo yum install -y nginx
mkdir /etc/nginx/sites-available
mkdir /etc/nginx/sites-enabled
sudo systemctl enable nginx
sudo systemctl start nginx
sudo systemctl enable nginx
sudo systemctl status nginx
### Python 3.5 ###
mkdir /usr/python3.5
./configure --prefix=/usr/python3.5
make
make install
/usr/python3.5/bin/pip3.5 install virtualenv
virtualenv --no-site-packages venv_py35 -p /usr/python3.5/bin/python3.5
/usr/python3.5/bin/virtualenv --no-site-packages venv_py35 -p /usr/python3.5/bin/python3.5
pip install flask schedule flask_httpauth pyDes pymysql
pip install future
### Launch Script ####
#!/bin/bash
PY_VENV='/mnt/home/dingjunnan/venv_py35'
NAME='credit_api'
HOME='/mnt/home/dingjunnan'
PROJECT_DIR=${HOME}/credit_api
SOCKFILE=${HOME}/sock/credit_gunicorn.sock
WORKERS=4
echo "starting $NAME as `whoami`"
source ${PY_VENV}/bin/activate
cd ${PROJECT_DIR}
gunicorn -b unix:${SOCKFILE} --name ${NAME} -w ${WORKERS} --max-requests 1000 --log-level=info wsgi:app
cloudera + CentOS7
# disable SELINUX
$ vi /etc/selinux/config
SELINUX=disabled
# disable firewalld
systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld
# User & Group
sudo groupadd cloudera-scm
sudo useradd --system --home=/opt/cm-5.12.1/run/cloudera-scm-server --shell=/bin/false --comment "Cloudera SCM User" -g cloudera-scm cloudera-scm
# chmod
sudo chown -R cloudera-scm:cloudera-scm /opt/cloudera
sudo chown -R cloudera-scm:cloudera-scm /opt/cm-5.12.1
# make dir
$sudo mkdir /var/lib/cloudera-scm-server
$sudo chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-server
$sudo mkdir /var/log/cloudera-scm-server
$sudo chown cloudera-scm:cloudera-scm /var/log/cloudera-scm-server
# hostname
$ sudo vi /etc/sysconfig/network
HOSTNAME=bigdatadev
$ sudo vi /etc/hosts
10.5.202.52 bigdatadev
$ hostname bigdatadev
$ /etc/init.d/network restart
# mysql
select @@validate_password_policy;
set global validate_password_policy=0;
set global validate_password_length=4;
ALTER USER 'root'@'localhost' IDENTIFIED BY '[email protected]';
GRANT ALL PRIVILEGES ON *.* TO [email protected]"%" IDENTIFIED BY "[email protected]";
flush privileges;
GRANT ALL ON *.* TO 'root'@'%' IDENTIFIED BY "[email protected]";
FLUSH PRIVILEGES;
SHOW GRANTS FOR 'bigdata'@'%';
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database reports_mgr DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
GRANT ALL PRIVILEGES ON *.* TO [email protected]"%";
sysctl -w net.ipv6.conf.all.disable_ipv6=1
sysctl -w net.ipv6.conf.default.disable_ipv6=1
./scm_prepare_database.sh mysql cm -hlocalhost -uroot [email protected] bigdatadev scm scm scmHive on spark configure
spark.executor.cores Between 5-7, See tuning details section
spark.executor.memory yarn.nodemanager.resource.memory-mb * (spark.executor.cores / yarn.nodemanager.resource.cpu-vcores)
30 * (5/40)
spark.yarn.executor.memoryOverhead 15-20% of spark.executor.memory
spark.executor.instances Depends on spark.executor.memory + spark.yarn.executor.memoryOverhead, see tuning details section.
set hive.execution.engine=spark;
set spark.executor.cores=5;
set spark.executor.memory=10GB;
set spark.yarn.executor.memoryOverhead=3GB;
set spark.executor.instances=20GB;
set hive.execution.engine=spark;
set spark.executor.cores=4;
set spark.executor.memory=13GB;
set spark.yarn.executor.memoryOverhead=2GB;
set spark.executor.instances=15GB;PYSPARK
#Horton .bashrc
export PYTHONPATH=${PYTHONPATH}:/usr/hdp/current/spark-client/python:/usr/hdp/current/spark-client/python/lib/py4j-0.9-src.zip
export SPARK_HOME=/usr/hdp/current/spark-client
#Cloudera .bashrc
export PYTHONPATH=${PYTHONPATH}:/opt/cloudera/parcels/SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354/lib/spark2/python:/opt/cloudera/parcels/SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354/lib/spark2/python/lib/py4j-0.10.4-src.zip
export SPARK_HOME=/opt/cloudera/parcels/SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354/lib/spark2
#Python packages
pip install findsparkvbox
wmic diskdrive list
wmic diskdrive list brief
VBoxManage internalcommands listpartitions -rawdisk \\.\PhysicalDrive1
C:\Program Files\Oracle\VirtualBox>VBoxManage.exe internalcommands createrawvmdk -filename F:\VirtualBoxRealDisk\PHYSICALDRIVE2.vmdk -rawdisk \\.\PHYSICALDRIVE2
C:\Program Files\Oracle\VirtualBox>VBoxManage.exe internalcommands createrawvmdk -filename C:\VirtualBoxRealDisk\PHYSICALDRIVE3P3.vmdk -rawdisk \\.\PHYSICALDRIVE3 -partitions 3superset
http://blog.csdn.net/dzjun/article/details/62421718
https://superset.incubator.apache.org/installation.html
https://github.com/apache/incubator-superset
Redash
https://blog.dshimizu.jp/article/225
command = “mysqldump -u%s -p%s –opt -R -d %s -h%s > %s” % (
server_user_name, server_user_passwd, server_db_schema, server_ip, des_path)
command = “mysqldump -u%s -p%s -t %s -h%s –triggers=false > %s” % (
server_user_name, server_user_passwd, server_db_schema, server_ip, des_path)
推荐阅读