10 Unstar Star 4 Fork 0

崔笑颜 / learningDocument

Create your Gitee Account
Explore and code with more than 5 million developers,Free private repositories !:)
Sign up
This repository doesn't specify license. Without author's permission, this code is only for learning and cannot be used for other purposes.
所有的学习资料,不在idea中的。 spread retract

Clone or download
软件安装.md 45.92 KB
Copy Edit Web IDE Raw Blame History
崔笑颜 authored 2020-09-25 16:40 . docker安装netdata

路径

数据库:/usr/local/database

服务器:/usr/local/serverFolder

mysql

# 拉取镜像
docker pull mysql:8.0.20
# 启动加挂载
docker run -d -p 3306:3306 -v /usr/local/database/mysql/conf:/etc/mysql/conf.d -v /usr/local/database/mysql/data:/var/lib/mysql -e MYSQL_ROOT_PASSWORD=123456 --name cxymysql mysql:8.0.20

集群

# 创建两个文件夹,用来挂载数据,和配置文件。
# 主节点
[mysqld]
server_id=1
skip-host-cache
skip-name-resolve
#  从节点
[mysqld]
server_id=2
skip-host-cache
skip-name-resolve
# 两个文件主要内容的差异是 server_id 值的不同,必须设置,否则在配置从库时,会报错:
# 主库启动
docker run -d -e MYSQL_ROOT_PASSWORD=123456 --name mysqlMaster --restart always -v /usr/local/database/mysql/mysqlMaster3306/data:/var/lib/mysql -v /usr/local/database/mysql/mysqlMaster3306:/etc/mysql/conf.d -p 3306:3306 mysql:8.0.20
# 从库启动
docker run -d -e MYSQL_ROOT_PASSWORD=123456 --name mysqlSlave --restart always -v /usr/local/database/mysql/mysqlSlave3307/data:/var/lib/mysql -v /usr/local/database/mysql/mysqlSlave3307:/etc/mysql/conf.d -p 3307:3306 mysql:8.0.20
# 启动成功,外部测试都可以连接成功
[root@centos8 mysqlSlave3307]# docker ps
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                               NAMES
992d0be96bc1        mysql:8.0.20        "docker-entrypoint.s…"   5 seconds ago       Up 3 seconds        33060/tcp, 0.0.0.0:3307->3306/tcp   mysqlSlave
23ceac64fab5        mysql:8.0.20        "docker-entrypoint.s…"   3 minutes ago       Up 3 minutes        0.0.0.0:3306->3306/tcp, 33060/tcp   mysqlMaster
# 进入主节点内部,查看主机挂载配置文件,没有问题。----------------------------------------------------------------
[root@centos8 mysqlSlave3307]# docker exec -it 23ceac64fab5 /bin/bash
root@23ceac64fab5:/# cd /etc/mysql/conf.d/
root@23ceac64fab5:/etc/mysql/conf.d# ls
data  master.cnf
root@23ceac64fab5:/etc/mysql/conf.d# cat master.cnf 
[mysqld]
server_id=1
skip-host-cache
skip-name-resolve
# 登陆主节点mysql 大p是端口,小p是密码
 mysql -u root -h 127.0.0.1 -P 3306 -p
# 创建用户
# 创建用户,identified with 'mysql_native_password' 表示使用该用户的密码加密方式。
# 赋予用户相应的权限;
# 刷新 MySql 系统权限。
 mysql> create user 'reader'@'%' identified with 'mysql_native_password' by 'reader';
Query OK, 0 rows affected (0.01 sec)

mysql> grant all privileges on *.* to 'reader'@'%';
Query OK, 0 rows affected (0.02 sec)

mysql> flush privileges;
Query OK, 0 rows affected (0.01 sec)
# 查看 master 节点的状态
mysql> show master status;
+---------------+----------+--------------+------------------+-------------------+
| File          | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set |
+---------------+----------+--------------+------------------+-------------------+
| binlog.000002 |      819 |              |                  |                   |
+---------------+----------+--------------+------------------+-------------------+
1 row in set (0.00 sec)
# 进入从节点内部,查看主机挂载配置文件,没有问题。-----------------------------------------------------------------
# 将次节点挂载到主库下面
mysql> CHANGE MASTER TO MASTER_HOST='192.168.106.129',Master_Port=3306,MASTER_USER='reader',MASTER_PASSWORD='reader',MASTER_LOG_FILE='binlog.000002',MASTER_LOG_POS=0;

Query OK, 0 rows affected, 2 warnings (0.05 sec)

# 开启主从同步
mysql> start slave;

Query OK, 0 rows affected (0.06 sec)
# 查看从库状态  Slave_IO_Running: Yes Slave_SQL_Running: Yes 即为成功。
mysql> show slave status\G;
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: 192.168.106.129
                  Master_User: reader
                  Master_Port: 3306
                Connect_Retry: 60
              Master_Log_File: binlog.000002
          Read_Master_Log_Pos: 819
               Relay_Log_File: 992d0be96bc1-relay-bin.000002
                Relay_Log_Pos: 1028
        Relay_Master_Log_File: binlog.000002
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes
              Replicate_Do_DB: 
          Replicate_Ignore_DB: 
           Replicate_Do_Table: 
       Replicate_Ignore_Table: 
      Replicate_Wild_Do_Table: 
  Replicate_Wild_Ignore_Table: 
                   Last_Errno: 0
                   Last_Error: 
                 Skip_Counter: 0
          Exec_Master_Log_Pos: 819
              Relay_Log_Space: 1244
              Until_Condition: None
               Until_Log_File: 
                Until_Log_Pos: 0
           Master_SSL_Allowed: No
           Master_SSL_CA_File: 
           Master_SSL_CA_Path: 
              Master_SSL_Cert: 
            Master_SSL_Cipher: 
               Master_SSL_Key: 
        Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
                Last_IO_Errno: 0
                Last_IO_Error: 
               Last_SQL_Errno: 0
               Last_SQL_Error: 
  Replicate_Ignore_Server_Ids: 
             Master_Server_Id: 1
                  Master_UUID: 887c1b23-d39c-11ea-ab3b-0242ac110002
             Master_Info_File: mysql.slave_master_info
                    SQL_Delay: 0
          SQL_Remaining_Delay: NULL
      Slave_SQL_Running_State: Slave has read all relay log; waiting for more updates
           Master_Retry_Count: 86400
                  Master_Bind: 
      Last_IO_Error_Timestamp: 
     Last_SQL_Error_Timestamp: 
               Master_SSL_Crl: 
           Master_SSL_Crlpath: 
           Retrieved_Gtid_Set: 
            Executed_Gtid_Set: 
                Auto_Position: 0
         Replicate_Rewrite_DB: 
                 Channel_Name: 
           Master_TLS_Version: 
       Master_public_key_path: 
        Get_master_public_key: 0
            Network_Namespace: 
1 row in set (0.00 sec)

ERROR: 
No query specified

image-20200801095951568

测试成功

image-20200801104306358

redis

# 拉取镜像
docker pull redis:6.0.5
# 先将配置文件放入到指定目录下,下面纯粹是在浪费篇幅,可以直接在百度下载。
# 启动
docker run --restart=always --log-driver json-file --log-opt max-size=100m --log-opt max-file=2 -p 6666:6666 --name cxyredis -v /usr/local/database/redis/redis.conf:/etc/redis/redis.conf -v /usr/local/database/redis:/data -d redis:6.0.5 redis-server /etc/redis/redis.conf --appendonly yes --requirepass "123456"

docker run --restart=always --log-driver json-file --log-opt max-size=100m --log-opt max-file=2 -p 6666:6666 --name cxyredis -v /usr/local/docker/redis/redis.conf:/etc/redis/redis.conf -v /usr/local/docker/redis:/data -d redis:6.0.5 redis-server /etc/redis/redis.conf --appendonly yes --requirepass "lauyang3959."

启动redis配置文件

文件在上级目录文件夹中。

哨兵

# 配置文件挂载一下,最低配置,内容如下 /etc/redis/sentinel.conf
sentinel monitor myredis 127.0.0.1 7000 1
# 创建一个哨兵容器
docker run --restart=always --log-driver json-file --log-opt max-size=100m --log-opt max-file=2 -p 7003:7003 --name redisSentinel -v /usr/local/database/redis/redis.conf:/etc/redis/redis.conf -v /usr/local/database/redis/redis-cluster/sentinel.conf:/etc/redis/sentinel.conf -d redis:6.0.5 redis-server /etc/redis/redis.conf --appendonly yes
# 访问	redis-sentinel /etc/redis/sentinel.conf
# 启动哨兵
[root@centos8 redis-cluster]# docker exec -it 48f8725736c3 redis-sentinel /etc/redis/sentinel.conf

主从配置

# 编写模版文件名为:redis-cluster.tmpl ,路径放在 /usr/local/database/redis/redis-cluster ---------------------
# 基本配置
protected-mode yes
port ${port}
bind 0.0.0.0 
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize no
supervised no
pidfile /var/run/redis_${port}.pid
loglevel notice
logfile ""
databases 16
always-show-logo yes
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump${port}.rdb
rdb-del-sync-files no
dir ./
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
replica-priority 100
acllog-max-len 128

 # requirepass 
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
lazyfree-lazy-user-del no


appendonly no

appendfilename "appendonly.aof"

appendfsync everysec


auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb


aof-load-truncated yes


aof-use-rdb-preamble yes

lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64


list-max-ziplist-size -2


list-compress-depth 0

set-max-intset-entries 512

zset-max-ziplist-entries 128
zset-max-ziplist-value 64

hll-sparse-max-bytes 3000

stream-node-max-bytes 4096
stream-node-max-entries 100

activerehashing yes
hz 10
dynamic-hz yes



# 创建配置脚本-----------------------------------------------------------------------------------------------

# 主目录
dir_redis_cluster='/usr/local/database/redis/redis-cluster'
# 节点地址号 从2开始
idx=1
# 逐个创建各节点目录和配置文件 三个
for port in `seq 7000 7002`; do
    # 创建存放redis数据路径
    mkdir -p ${dir_redis_cluster}/${port}/data;
    # 通过模板个性化各个节点的配置文件
    idx=$(($idx+1));
    port=${port} ip=`echo ${gateway} | sed "s/1$/$idx/g"` \
        envsubst < ${dir_redis_cluster}/redis-cluster.tmpl \
        > ${dir_redis_cluster}/${port}/redis-${port}.conf
done
# 配置并启动-----------------------------------------------------------------------------------------------
# 创建容器配置并运行 redis.conf后面你的版本,默认是最新。
for port in `seq 7000 7002`; do
    docker run --name redis-${port} --net redis-cluster-net -d \
        -p ${port}:${port} -p 1${port}:1${port} \
        -v ${dir_redis_cluster}/${port}/data:/data \
        -v ${dir_redis_cluster}/${port}/redis-${port}.conf:/usr/local/etc/redis/redis.conf redis \
        redis-server /usr/local/etc/redis/redis.conf
done
# 查看集群功能是否开启,这里需要让它不成功 info cluster-----------------------------------------------------------
[root@centos8 ~]# docker exec -it redis-7000 redis-cli -p 7000 info cluster
# Cluster
cluster_enabled:0
# 其他操作(注意自己的路径)--------------------------------------------------------------------------
#!/bin/bash
# 外部输入命令
com=$1
# 主目录
dir_redis_cluster='/usr/local/database/redis/redis-cluster'
# redis集群网关
gateway='172.18.0.1'

case ${com} in
	create)
        idx=1;
		for port in `seq 7000 7005`; do
            # 创建存放redis数据路径
			mkdir -p ${dir_redis_cluster}/${port}/data;
            # 通过模板个性化各个节点的配置文件
            idx=$(($idx+1));
            port=${port} ip=`echo ${gateway} | sed "s/1$/$idx/g"` \
                envsubst < ${dir_redis_cluster}/redis-cluster.tmpl \
                > ${dir_redis_cluster}/${port}/redis-${port}.conf
		done
	;;
    build)
        # 创建容器配置并运行
        for port in `seq 7000 7005`; do
            docker run --name redis-${port} --net redis-cluster-net -d \
            	-p ${port}:${port} -p 1${port}:1${port} \
                -v ${dir_redis_cluster}/${port}/data:/data \
                -v ${dir_redis_cluster}/${port}/redis-${port}.conf:/usr/local/etc/redis/redis.conf redis \
                redis-server /usr/local/etc/redis/redis.conf
        done
    ;;
    start | begin)
        # 运行容器
    	for port in `seq 7000 7002`; do
            docker start redis-${port}
        done
    ;;
    stop | end)
        # 停止容器运行
        for port in `seq 7000 7002`; do
            docker stop redis-${port}
        done
    ;;
    rm)
        # 删除已有容器
        for port in `seq 7000 7002`; do
            docker rm redis-${port}
        done
    ;;
    restart)
        # 重启已有容器
    	for port in `seq 7000 7002`; do
            docker restart redis-${port}
        done
    ;;
    destroy)
        # 删除集群目录及配置
        for port in `seq 7000 7002`; do
            rm -rf ${dir_redis_cluster}/${port}
        done
    ;;
    *)
        echo "Usage:	./build [create|build|start|stop|rm|restart|destroy]"
    ;;
esac
# 重点如下。-------------------------------------------------------------------------------------------------
[root@centos8 redis-cluster]# docker ps
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                                                        NAMES
2157b7f9d97d        redis               "docker-entrypoint.s…"   5 seconds ago       Up 4 seconds        0.0.0.0:7002->7002/tcp, 6379/tcp, 0.0.0.0:17002->17002/tcp   redis-7002
f1ac84714bf7        redis               "docker-entrypoint.s…"   6 seconds ago       Up 4 seconds        0.0.0.0:7001->7001/tcp, 6379/tcp, 0.0.0.0:17001->17001/tcp   redis-7001
0db76b9cbe64        redis               "docker-entrypoint.s…"   6 seconds ago       Up 5 seconds        0.0.0.0:7000->7000/tcp, 6379/tcp, 0.0.0.0:17000->17000/tcp   redis-7000
# 主机
127.0.0.1:7000> info replication #查看主从信息
# Replication
role:master
connected_slaves:2
slave0:ip=172.18.0.1,port=7001,state=online,offset=392,lag=0
slave1:ip=172.18.0.1,port=7002,state=online,offset=392,lag=1
master_replid:e2c07eadf81344899ffd7cec60036686885c2947
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:392
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:392
# 从机,得用ip,我真是服了。就算是本机127也不行。SLAVEOF 192.168.106.129 7000
[root@centos8 ~]# docker exec -it redis-7001 /bin/bash
root@cb475fe3b9e0:/data# redis-cli -h 192.168.106.129 -p 7001
192.168.106.129:7001> info replication
# Replication
role:master
connected_slaves:0
master_replid:868aadcb711e66461f5cd50b5ef59cf9cacbb26c
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:0
second_repl_offset:-1
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0
192.168.106.129:7001> SLAVEOF 192.168.106.129 7000 #设置为从机,二号机也是一样。
OK
192.168.106.129:7001> info replication 
# Replication
role:slave
master_host:192.168.106.129
master_port:7000
master_link_status:up
master_last_io_seconds_ago:3
master_sync_in_progress:0
slave_repl_offset:336
slave_priority:100
slave_read_only:1
connected_slaves:0
master_replid:e2c07eadf81344899ffd7cec60036686885c2947
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:336
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:336

集群模式

# 普通方式
port 6378
pidfile /var/run/redis_6378.pid
logfile "/dev/redis78.log"
dbfilename dump6378.rdb
# docker方式-----------------------------------------------------------------------------------------------
# 创建属于redis的集群网络------------------------------------------------------------------------------------
docker network create redis-cluster-net
# 查看ip 172.18.0.0----------------------------------------------------------------------------------------
[root@centos8 ~]# docker network inspect redis-cluster-net
[
    {
        "Name": "redis-cluster-net",
        "Id": "c3550af56a4c5a4894e04963b5b747361122e3666f58bd1e88e854f4b295f316",
        "Created": "2020-07-29T04:42:39.479861603-04:00",
        "Scope": "local",
        "Driver": "bridge",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": {},
            "Config": [
                {
                    "Subnet": "172.18.0.0/16",
                    "Gateway": "172.18.0.1"
                }
            ]
        },
        "Internal": false,
        "Attachable": false,
        "Ingress": false,
        "ConfigFrom": {
            "Network": ""
        },
        "ConfigOnly": false,
        "Containers": {
            "2d6356126ef5b579a189af1e89e8b777cdb12f538a2e03bb8816e4308a20490f": {
                "Name": "redis-7002",
                "EndpointID": "183d44748f7586bc812b0a714678de09cc7b931b8f08fcc5d24bfb303a1893d3",
                "MacAddress": "02:42:ac:12:00:04",
                "IPv4Address": "172.18.0.4/16",
                "IPv6Address": ""
            },
            "32525d24f5264aebe9bcabe20c5ae9456b4f9a9945d08e622fe65bb5818b513a": {
                "Name": "redis-7000",
                "EndpointID": "262490e1bd204de27db1c22128efacee6bff77e49aefae24df8c6fb8b76d0744",
                "MacAddress": "02:42:ac:12:00:02",
                "IPv4Address": "172.18.0.2/16",
                "IPv6Address": ""
            },
            "50403e09aae463ff61a2a23621eb1d1b93b7fa40f0edb12b279d79fbba9eea4b": {
                "Name": "redis-7001",
                "EndpointID": "c10219728746f7e328d3528f56dcf011b0f8fbc42f628a05dd377146bb0d9d85",
                "MacAddress": "02:42:ac:12:00:03",
                "IPv4Address": "172.18.0.3/16",
                "IPv6Address": ""
            },
            "637d28ddb99c7bf82fcfb222d0244918b43977a220e8f61caea264c4aa6e97ea": {
                "Name": "redis-7003",
                "EndpointID": "4b6ac8bdd02c91019b97989c78d411f966cce93f2833e799be1570f065720499",
                "MacAddress": "02:42:ac:12:00:05",
                "IPv4Address": "172.18.0.5/16",
                "IPv6Address": ""
            },
            "b0eeb13258499f64a5d4308dbbbbef53747af9a185c01af78f1f0c597b299d99": {
                "Name": "redis-7005",
                "EndpointID": "53be06c5a30cb88848edd2e57b3c559b701d03a9aab541dd414d6ba38240a99f",
                "MacAddress": "02:42:ac:12:00:07",
                "IPv4Address": "172.18.0.7/16",
                "IPv6Address": ""
            },
            "f954451e3775a9921566d6b91007ff196eeb385f4ee4e12a91bb14dda14dabed": {
                "Name": "redis-7004",
                "EndpointID": "da9bac581d4cc0586c11ecd2eaceb6408fe45f06b94abf99f85a31cc44c002ba",
                "MacAddress": "02:42:ac:12:00:06",
                "IPv4Address": "172.18.0.6/16",
                "IPv6Address": ""
            }
        },
        "Options": {},
        "Labels": {}
    }
]
# 编写模版文件名为:redis-cluster.tmpl ,路径放在 /usr/local/database/redis/redis-cluster ---------------------
# 基本配置
## 开放端口
port ${port}
## 不作为守护进程
daemonize no
## 启用aof持久化模式
appendonly yes

# 集群配置
## 开启集群配置
cluster-enabled yes
## 存放集群节点的配置文件 系统自动建立
cluster-config-file nodes-${port}.conf
## 节点连接超时时间
cluster-node-timeout 50000  
## 实际为各节点网卡分配ip
cluster-announce-ip ${ip}
## 节点映射端口
cluster-announce-port ${port}
## 节点总线端口
cluster-announce-bus-port 1${port}
cluster-slave-validity-factor 10
cluster-migration-barrier 1
cluster-require-full-coverage yes

# 创建配置脚本-----------------------------------------------------------------------------------------------

# 主目录
dir_redis_cluster='/usr/local/database/redis/redis-cluster'
# docker redis集群网关
gateway='172.18.0.1'
# 节点地址号 从2开始
idx=1
# 逐个创建各节点目录和配置文件
for port in `seq 7000 7005`; do
    # 创建存放redis数据路径
    mkdir -p ${dir_redis_cluster}/${port}/data;
    # 通过模板个性化各个节点的配置文件
    idx=$(($idx+1));
    port=${port} ip=`echo ${gateway} | sed "s/1$/$idx/g"` \
        envsubst < ${dir_redis_cluster}/redis-cluster.tmpl \
        > ${dir_redis_cluster}/${port}/redis-${port}.conf
done
# 配置并启动-----------------------------------------------------------------------------------------------
# 创建容器配置并运行 redis.conf后面你的版本,默认是最新。
for port in `seq 7000 7005`; do
    docker run --name redis-${port} --net redis-cluster-net -d \
        -p ${port}:${port} -p 1${port}:1${port} \
        -v ${dir_redis_cluster}/${port}/data:/data \
        -v ${dir_redis_cluster}/${port}/redis-${port}.conf:/usr/local/etc/redis/redis.conf redis \
        redis-server /usr/local/etc/redis/redis.conf
done
# 查看集群功能是否开启 info cluster--------------------------------------------------------------------------
[root@centos8 ~]# docker exec -it redis-7000 redis-cli -p 7000 info cluster
# Cluster
cluster_enabled:1
# 节点连接,一条条执行 --------------------------------------------------------------------------
docker exec -it redis-7000 redis-cli -p 7000 cluster meet 172.18.0.3 7001
docker exec -it redis-7000 redis-cli -p 7000 cluster meet 172.18.0.4 7002
docker exec -it redis-7000 redis-cli -p 7000 cluster meet 172.18.0.5 7003
docker exec -it redis-7000 redis-cli -p 7000 cluster meet 172.18.0.6 7004
docker exec -it redis-7000 redis-cli -p 7000 cluster meet 172.18.0.7 7005
# 进入7000执行 cluster nodes --------------------------------------------------------------------------
127.0.0.1:7000> cluster nodes
33cc63c86da4d85152990cf30534ced11b7abd73 172.18.0.2:7000@17000 myself,master - 0 1596013060000 1 connected
5137b18fe4f3d975c1513f5761ffa8b9615ff077 172.18.0.3:7001@17001 master - 0 1596013059000 2 connected
a61eab3d027b9ee4f521717cfba37b5ef8f19ad5 172.18.0.4:7002@17002 master - 0 1596013060598 3 connected
f543363850c50bb90cd34694e4dc7235df4ea399 172.18.0.5:7003@17003 master - 0 1596013061613 0 connected
955809a4a1a643719eafd3fc04f98ed9800cacb6 172.18.0.6:7004@17004 master - 0 1596013059587 4 connected
10e55a0222db69b711e146ef134adb45725693c1 172.18.0.7:7005@17005 master - 0 1596013058000 5 connected
# 设置主从节点,注意对应你自己的节点--------------------------------------------------------------------------
# 设置7001节点为7000节点的从节点
docker exec -it redis-7001 redis-cli -p 7001 cluster replicate 33cc63c86da4d85152990cf30534ced11b7abd73 # 7001 --> 7000
# 设置7003节点为7002节点的从节点
docker exec -it redis-7003 redis-cli -p 7003 cluster replicate a61eab3d027b9ee4f521717cfba37b5ef8f19ad5 # 7003 --> 7002
# 设置7005节点为7004节点的从节点
docker exec -it redis-7005 redis-cli -p 7005 cluster replicate 955809a4a1a643719eafd3fc04f98ed9800cacb6 # 7005 --> 7004
# 将16384个槽分配到3个主节点去, 每个节点平均分的5461个槽---------------------------------------------------------
# 7000 0~5460
docker exec -it redis-7000 redis-cli -p 7000 cluster addslots {0..5460}
# 7002 5461~10920
docker exec -it redis-7002 redis-cli -p 7002 cluster addslots {5461..10920}
# 7004 10920~16383
docker exec -it redis-7004 redis-cli -p 7004 cluster addslots {10921..16383}
# 测试------------------------------------------------------------------------------------------------------
# cluster slots
127.0.0.1:7000> cluster slots
1) 1) (integer) 0
   2) (integer) 5460
   3) 1) "172.18.0.2"
      2) (integer) 7000
      3) "33cc63c86da4d85152990cf30534ced11b7abd73"
   4) 1) "172.18.0.3"
      2) (integer) 7001
      3) "5137b18fe4f3d975c1513f5761ffa8b9615ff077"
2) 1) (integer) 10921
   2) (integer) 16383
   3) 1) "172.18.0.6"
      2) (integer) 7004
      3) "955809a4a1a643719eafd3fc04f98ed9800cacb6"
   4) 1) "172.18.0.7"
      2) (integer) 7005
      3) "10e55a0222db69b711e146ef134adb45725693c1"
3) 1) (integer) 5461
   2) (integer) 10920
   3) 1) "172.18.0.4"
      2) (integer) 7002
      3) "a61eab3d027b9ee4f521717cfba37b5ef8f19ad5"
   4) 1) "172.18.0.5"
      2) (integer) 7003
      3) "f543363850c50bb90cd34694e4dc7235df4ea399"

# 其他操作(注意自己的路径)--------------------------------------------------------------------------
#!/bin/bash
# 外部输入命令
com=$1
# 主目录
dir_redis_cluster='/usr/local/database/redis/redis-cluster'
# redis集群网关
gateway='172.18.0.1'

case ${com} in
	create)
        idx=1;
		for port in `seq 7000 7005`; do
            # 创建存放redis数据路径
			mkdir -p ${dir_redis_cluster}/${port}/data;
            # 通过模板个性化各个节点的配置文件
            idx=$(($idx+1));
            port=${port} ip=`echo ${gateway} | sed "s/1$/$idx/g"` \
                envsubst < ${dir_redis_cluster}/redis-cluster.tmpl \
                > ${dir_redis_cluster}/${port}/redis-${port}.conf
		done
	;;
    build)
        # 创建容器配置并运行
        for port in `seq 7000 7005`; do
            docker run --name redis-${port} --net redis-cluster-net -d \
            	-p ${port}:${port} -p 1${port}:1${port} \
                -v ${dir_redis_cluster}/${port}/data:/data \
                -v ${dir_redis_cluster}/${port}/redis-${port}.conf:/usr/local/etc/redis/redis.conf redis \
                redis-server /usr/local/etc/redis/redis.conf
        done
    ;;
    start | begin)
        # 运行容器
    	for port in `seq 7000 7005`; do
            docker start redis-${port}
        done
    ;;
    stop | end)
        # 停止容器运行
        for port in `seq 7000 7005`; do
            docker stop redis-${port}
        done
    ;;
    rm)
        # 删除已有容器
        for port in `seq 7000 7005`; do
            docker rm redis-${port}
        done
    ;;
    restart)
        # 重启已有容器
    	for port in `seq 7000 7005`; do
            docker restart redis-${port}
        done
    ;;
    destroy)
        # 删除集群目录及配置
        for port in `seq 7000 7005`; do
            rm -rf ${dir_redis_cluster}/${port}
        done
    ;;
    *)
        echo "Usage:	./build [create|build|start|stop|rm|restart|destroy]"
    ;;
esac

# 最终效果
[root@centos8 ~]# docker exec -it redis-7000 redis-cli -c -p 7000
127.0.0.1:7000> set name 1
-> Redirected to slot [5798] located at 172.18.0.4:7002
OK
172.18.0.4:7002> keys *
1) "name"
172.18.0.4:7002> set age 18
-> Redirected to slot [741] located at 172.18.0.2:7000
OK

# java测试
public class TestRedisConnect {
    @Test
    public void connectCluster() {
        Set<HostAndPort> nodes = new HashSet<>();
        nodes.add(new HostAndPort("127.0.0.1", 7000));
        nodes.add(new HostAndPort("127.0.0.1", 7001));
        nodes.add(new HostAndPort("127.0.0.1", 7002));
        nodes.add(new HostAndPort("127.0.0.1", 7003));
        nodes.add(new HostAndPort("127.0.0.1", 7004));
        nodes.add(new HostAndPort("127.0.0.1", 7005));

        JedisCluster cluster = new JedisCluster(nodes, 5000);

        System.out.println(cluster.get("hello"));

        cluster.set("test2", "6739");
        System.out.println(cluster.get("test2"));

        Map<String, String> inviteePhone = new HashMap<>(5);
        inviteePhone.put("inviterID", "1001");
        inviteePhone.put("status", "0");
        // hash表 批处理
        cluster.hmset("inviteePhone", inviteePhone);

        System.out.println(cluster.hget("inviteePhone", "inviterID"));
        System.out.println(cluster.hget("inviteePhone", "status"));
    }
}

image-20200722164014516

nginx

# 拉取镜像
docker pull nginx:1.19.0

# 编辑nginx.conf
vim /usr/local/serverFolder/nginx/conf/nginx.conf

user  nginx;
worker_processes  1;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}


# 编辑default.conf
vim /usr/local/serverFolder/nginx/conf.d/default.conf

server {
    listen       80;
    server_name  localhost;

    #charset koi8-r;
    #access_log  /var/log/nginx/host.access.log  main;

    location / {
        root   /usr/share/nginx/html;
        index  index.html index.htm;
    }

    #error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }
}


# 启动
docker run --name cxynginx -d -p 80:80  -v /usr/local/serverFolder/nginx/html:/usr/share/nginx/html -v /usr/local/serverFolder/nginx/conf/nginx.conf:/etc/nginx/nginx.conf  -v /usr/local/serverFolder/nginx/logs:/var/log/nginx -v /usr/local/serverFolder/nginx/conf.d:/etc/nginx/conf.d nginx:1.19.0

域名

# 配置域名,前后不分离。
server {
	    listen 80;
	    server_name bk.cuixiaoyan.xyz www.bk.cuixiaoyan.xyz;
	    
	    # listen 443 ssl;
		  # ssl off;
		  # ssl_certificate ../cert/zhzq1.pem;
		  # ssl_certificate_key ../cert/zhzq1.key;
		  # ssl_session_timeout 5m;
		  # ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE:ECDH:AES:HIGH:!NULL:!aNULL:!MD5:!ADH:!RC4;
		  # ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
		  # ssl_prefer_server_ciphers on;
	    location / {
	    	   proxy_pass http://127.0.0.1:888/;
	        proxy_set_header HOST $host;
	        proxy_set_header X-Forwarded-Proto $scheme;
	        proxy_set_header X-Real-IP $remote_addr;
	        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
	    }
	    error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
	}
	
# 配置域名,前后分离。
    server {
        listen 80;
        server_name cuixiaoyan.xyz www.cuixiaoyan.xyz;
	   
       listen 443 ssl;
        ssl off;
        ssl_certificate ../cert/zhzq.pem;
        ssl_certificate_key ../cert/zhzq.key;
        ssl_session_timeout 5m;
        ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE:ECDH:AES:HIGH:!NULL:!aNULL:!MD5:!ADH:!RC4;
        ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
        ssl_prefer_server_ciphers on;
	gzip on;
    gzip_min_length 1k;
    gzip_comp_level 9;
    gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;
    gzip_vary on;
    gzip_disable "MSIE [1-6]\.";
        location ^~ /jeecg-boot {
            proxy_pass              http://127.0.0.1:8080/jeecg-boot/;
            proxy_set_header        Host 127.0.0.1;
            proxy_set_header        X-Real-IP $remote_addr;
            proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
        }
        location / {
            root   html;
            index  index.html index.htm;
	    if (!-e $request_filename) {
                rewrite ^(.*)$ /index.html?s=$1 last;
                break;
            }
    }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }

tomcat

# 拉取镜像
docker pull tomcat:9.0
# 启动并挂载
docker run -d --name=cxytomcat9 -v /usr/local/serverFolder/tomcat9/webapps:/usr/local/tomcat/webapps --privileged=true -p 8080:8080 tomcat:9.0
# 首页设置
# 进入容器
docker exec -it 容器id /bin/bash
# 将webapps.dist下拷贝到webapps下
cp -r webapps.dist/* webapps

image-20200628142022006

Elasticsearch Kibana

安装elasticsearch

# 拉取镜像
docker pull elasticsearch:7.6.2
# 启动镜像,设置内存大小,注意与Kibana版本保持一致,后期springBoot也是。
docker run --name cxyes7.6 -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms64m -Xmx512m" -d elasticsearch:7.6.2
# 测试安装
curl http://localhost:9200

image-20200708170840798

安装Kibana

# 拉取镜像
docker pull kibana:7.6.2
# 启动镜像
docker run --name cxykibana --link cxyes7.6:elasticsearch -p 5601:5601 -d kibana:7.6.2
# 设置中文,/usr/share/kibana/config/kibana.yml
docker exec -it cxykibana /bin/bash
cd config/
vi kibana.yml
#  在末尾添加i18n.locale: zh-CN即可. 注意冒号后一定要有空格!
i18n.locale: zh-CN
# 然后重启
docker restart cxykibana

image-20200708172652859

安装可视化head

# 拉取镜像
docker pull mobz/elasticsearch-head:5
# 启动镜像
docker run -d  --name="cxyhead" -p 9100:9100 b19a5c98e43b
# 启动成功
[root@centos8 ~]# docker run -d  --name="cxyhead" -p 9100:9100 b19a5c98e43b
2a599c351909d52b866277de8e2dd3775802285628200747fbad3a7780f4ee91

image-20200712091855753

image-20200712091923476

这里查询如果查询不到数据的话,需要设置如下

image-20200712101147355

# 进入head容器   这里容器id输入自己的
docker exec -it b19a5c98e43b /bin/bash

# 编辑配置文件,如果没有vim命令自行安装vim
vim _site/vendor.js

# 在未编辑的模式下使用以下命令查询
/contentType
输入i开启编辑模式,application/x-www-form-urlencoded 改为 application/json;charset=UTF-8

# 按键盘的ESC健退出编辑模式,然后查询下一个
/var inspectData = s.contentType
输入i开启编辑模式,将application/x-www-form-urlencoded 改为 application/json;charset=UTF-8

# 输入wq保存并退出,最后退出容器并重启容器
exit
docker restart b19a5c98e43b

其他设置

  1. 解决跨域访问

首先进入到ES容器中, 然后进入到指定目录修改elasticsearch.yml文件

docker exec -it cxyes7.6 /bin/bash
cd config/
vi elasticsearch.yml
# 在elasticsearch.yml的文件末尾加上
http.cors.enabled: true
http.cors.allow-origin: "*"
# 修改配置后重启容器即可
docker restart cxyes7.6
  1. 安装IK中文分词器

ES自带的分词器对中文分词不友好, 所以我们下载开源的IK分词器来解决这个问题. 首先进入到plugins目录中下载分词器, 下载完成后解压, 再重启ES即可. 具体步骤如下:

image-20200708173403601

docker exec -it cxyes7.6 /bin/bash
# 将文件拷入。本地安装方式,网速慢时使用。
docker cp ./elasticsearch-analysis-ik-7.6.2 6ef22c7dad68:/usr/share/elasticsearch/plugins
# 进入插件目录
cd plugins/
# 下载分词器,在线安装。
elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.6.2/elasticsearch-analysis-ik-7.6.2.zip

elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v6.8.3/elasticsearch-analysis-ik-6.8.3.zip
# 重启es
docker restart cxyes7.6
# 测试
POST _analyze
{
  "analyzer": "ik_smart",
  "text": "各个国家有各个国家的国歌"
}

然后重启容器, 可以在Kibana界面的dev tools中验证是否安装成功

image-20200708173545327

nacos

# 拉取官网最新版镜像,如果嫌大的话,也可以自己制作,或者使用其他大佬制作的。
docker pull nacos/nacos-server
# 这里我嫌大,就下载了下面这个。
docker pull jerry6290/nacos-server
# 启动命令
docker run --env MODE=standalone --name cxynacos -d -p 8848:8848 镜像id
# 然后访问,ip+端口+/nacos
# 进入容器,如果 /bin/bash 找不到,换成如下命令。
docker exec -it 容器id /bin/sh
# 增加数据库持久化,如下图,然后重启nacos。

image-20200725195845781

image-20200725200018801

image-20200725203949014

最后效果

image-20200725204150009

sentinel-dashboard

# 拉取镜像
docker pull bladex/sentinel-dashboard
# 启动,这也太简单了吧。
docker run --name cxysentinel -d -p 8858:8858 -d 镜像id

image-20200726101905251

springBoot项目

创建Dockerfile

名字只能是Dockerfile,下面的app.jar按照自己的名字写。或者将自己的jar包名字改成这个。

image-20200728172034258

打包

image-20200728172200435

上传到服务器

[root@centos8 esDocker]# docker build -t es:1.0  . #打包命令,注意点。
Sending build context to Docker daemon  52.22MB
Step 1/5 : FROM java:8
8: Pulling from library/java
5040bd298390: Pull complete 
fce5728aad85: Pull complete 
76610ec20bf5: Pull complete 
60170fec2151: Pull complete 
e98f73de8f0d: Pull complete 
11f7af24ed9c: Pull complete 
49e2d6393f32: Pull complete 
bb9cdec9c7f3: Pull complete 
Digest: sha256:c1ff613e8ba25833d2e1940da0940c3824f03f802c449f3d1815a66b7f8c0e9d
Status: Downloaded newer image for java:8
 ---> d23bdf5b1b1b
Step 2/5 : COPY *.jar /app.jar
 ---> 815e6fde77ae
Step 3/5 : CMD ["--server.port=8080"]
 ---> Running in d51406f8c79f
Removing intermediate container d51406f8c79f
 ---> 83a3ad00809a
Step 4/5 : EXPOSE 8080
 ---> Running in e383231466d2
Removing intermediate container e383231466d2
 ---> f1b82f641b8a
Step 5/5 : ENTRYPOINT ["java","-jar","app.jar"]
 ---> Running in ace034ac06de
Removing intermediate container ace034ac06de
 ---> 4783629031ef
Successfully built 4783629031ef
Successfully tagged es:1.0
[root@centos8 esDocker]# docker images #查看镜像
REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
es                  1.0                 4783629031ef        10 seconds ago      695MB
java                8                   d23bdf5b1b1b        3 years ago         643MB
[root@centos8 esDocker]# docker run -d -p 8080:8080 --name cxyes1.0 4783629031ef # 启动镜像
10c397ad2e7610e0d5d018a8a45325dffaad647e62ca49e764c0d742994e16d5
[root@centos8 esDocker]# docker ps # 查看运行的镜像。
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                    NAMES
10c397ad2e76        4783629031ef        "java -jar app.jar -…"   3 minutes ago       Up 3 minutes        0.0.0.0:8080->8080/tcp   cxyes1.0

image-20200729114155455

MinIo

# 拉镜像
docker pull minio/minio
# 启动镜像
docker run -p 9000:9000 --name cxyMinio -v /usr/local/docker/minio/localtime:/etc/localtime -v /usr/local/docker/minio/data:/data -v /usr/local/docker/minio/config:/root/.minio -d minio/minio server /data

默认密码都是 minioadmin

创建一个存储桶

image-20200815143347827

设置权限

要不然访问不到图片 image-20200815143431731

zookeeper

# 拉取镜像
docker pull zookeeper
# 启动镜像
docker run -d -p 2181:2181 --name cxyZookeeper --restart always 镜像id
# 启动
docker exec -it 容器id ./bin/zkCli.sh
image-20200816152922826

consul

# 拉取
docker pull consul 
# 启动
docker run -d -p 8500:8500 -v /usr/local/docker/consul:/consul/data -e CONSUL\_BIND\_INTERFACE='eth0' --name=cxyConsul 镜像id agent -server -bootstrap -ui -node=node-1 -client='0.0.0.0' -datacenter=cxy

server: 以server身份启动。默认是client
bootstrap-expect:集群要求的最少server数量,当低于这个数量,集群即失效。
data-dir:data存放的目录,更多信息请参阅consul数据同步机制
node:节点id,集群中的每个node必须有一个唯一的名称。默认情况下,Consul使用机器的hostname
bind:监听的ip地址。默认绑定0.0.0.0,可以不指定。表示Consul监听的地址,而且它必须能够被集群中的其他节点访问。Consul默认会监听第一个private IP,但最好还是提供一个。生产设备上的服务器通常有好几个网卡,所以指定一个不会出错
client: 客户端的ip地址,0.0.0.0是指谁都可以访问(不加这个,下面的ui :8500无法访问)
ui: 可以访问UI界面
-config-dir指定配置文件夹,Consul会加载其中的所有文件
-datacenter 指定数据中心名称,默认是dc1

image-20200816161348538

Jenkins

根据镜像OFFICIAL标签判断,位列第一、名字为jenkins的是 Jenkins 官方提供的镜像。我们是不是应该使用这个官方镜像呢?

如果你看过 Jenkins 文档中 Docker 安装 小节,会发现官方推荐使用的镜像是jenkinsci/blueocean,该镜像包含当前的长期支持 (LTS) 的 Jenkins 版本 (可以生产使用) ,并捆绑了所有 Blue Ocean 插件和功能。

第一次安装的时候使用的是jenkins镜像,其集成的 Jenkins 版本比较高,结果出现好多插件不兼容的情况,对于我等小白来说是一件比较麻烦的事。

这里我们果断使用jenkinsci/blueocean镜像,使用docker pull命令拉取:

# 拉取镜像
docker pull jenkinsci/blueocean
# 启动容器
docker run \
  --name cxyJenkins-blueocean \
  -d \
  -p 8081:8080 \
  -p 50000:50000 \
  -v /usr/local/docker/jenkins:/var/jenkins_home \
  jenkinsci/blueocean
 # 报错
 [root@centos8 jenkins]# docker logs 06a6e888dbf5
touch: cannot touch '/var/jenkins_home/copy_reference_file.log': Permission denied
Can not write to /var/jenkins_home/copy_reference_file.log. Wrong volume permissions?
# 修改
sudo chown -R 1000:1000 /usr/local/docker/jenkins
# 成功
root@centos8 jenkins]# sudo chown -R 1000:1000 /usr/local/docker/jenkins
[root@centos8 jenkins]# docker start 06a6e888dbf5
06a6e888dbf5
[root@centos8 jenkins]# docker ps 
CONTAINER ID        IMAGE                 COMMAND                  CREATED             STATUS              PORTS                                                  NAMES
06a6e888dbf5        jenkinsci/blueocean   "/sbin/tini -- /usr/…"   4 minutes ago       Up 3 seconds        0.0.0.0:50000->50000/tcp, 0.0.0.0:8081->8080/tcp       cxyJenkins-blueocean
63785bef8658        6ad6cb039dfa          "/docker-entrypoint.…"   2 weeks ago         Up 10 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2181->2181/tcp, 8080/tcp   cxyZookeeper
992d0be96bc1        mysql:8.0.20          "docker-entrypoint.s…"   5 weeks ago         Up 10 minutes       33060/tcp, 0.0.0.0:3307->3306/tcp                      mysqlSlave
23ceac64fab5        mysql:8.0.20          "docker-entrypoint.s…"   5 weeks ago         Up 10 minutes       0.0.0.0:3306->3306/tcp, 33060/tcp                      mysqlMaster
48f8725736c3        redis:6.0.5           "docker-entrypoint.s…"   5 weeks ago         Up 10 minutes       6379/tcp, 0.0.0.0:7003->7003/tcp                       redisSentinel
38c2f6e00354        redis:6.0.5           "docker-entrypoint.s…"   5 weeks ago         Up 10 minutes       6379/tcp, 0.0.0.0:6666->6666/tcp                       cxyredis
# 访问 以上如果一切正常,现在就可以通过http://<ipaddress>:8081 访问 Jenkins 的 Web 界面了。
# 管理员初始化密码
[root@centos8 jenkins]# docker exec -it 06a6e888dbf5 /bin/bash
bash-5.0$ cat /var/jenkins_home/secrets/initialAdminPassword
c994db795bb9450399b48e54efbc1b47

rabbitmq

# 拉取镜像,官网版本的不带管理界面,安装这个比较方便
docker pull docker.io/rabbitmq:3.8-management
# 启动镜像
docker run --name cxyRabbitmq -d -p 15672:15672 -p 5672:5672 587380cbba10
image-20200905191609521

zipkin

# 拉取镜像
docker pull openzipkin/zipkin
# 启动
docker run --name cxyZipkin -d -p 9411:9411 openzipkin/zipkin
image-20200906083946587

seata-server

# 拉取镜像
docker pull seataio/seata-server:1.0.0
# 启动并挂载
docker run --name cxySeata-server -e SEATA_IP=192.168.106.129 -d -p 8091:8091 seataio/seata-server:1.0.0
# 修改配置
docker exec -it 镜像id /bin/sh
# 修改默认名称,和db数据库连接。数据库文件,容器里面没有,需要到官网下载。
image-20200913100727812

netdata

# 拉取镜像
docker pull netdata/netdata
# 启动
docker run --name cxyNetdata -d --cap-add SYS_PTRACE -v /usr/local/docker/netdata/proc:/host/proc:ro  -v /usr/local/docker/netdata/sys:/host/sys:ro  -p 19999:19999 --restart unless-stopped netdata/netdata
# 汉化文件在上层 netdata-cpatch 中,移动到指定目录。
[root@centos8 Sinicization]# docker cp ./dashboard_info.js 537f44f4418f:/usr/share/netdata/web/
[root@centos8 Sinicization]# docker cp ./dashboard.js 537f44f4418f:/usr/share/netdata/web/
[root@centos8 Sinicization]# docker cp ./index.html 537f44f4418f:/usr/share/netdata/web/
[root@centos8 Sinicization]# docker cp ./main.js 537f44f4418f:/usr/share/netdata/web/
# 不需要重启,但是需要清理浏览器的缓存。
  

image-20200925150851672

Comment ( 0 )

Sign in for post a comment

1
https://gitee.com/cuixiaoyan/learningDocument.git
git@gitee.com:cuixiaoyan/learningDocument.git
cuixiaoyan
learningDocument
learningDocument
master

Search

132457 8cb2edc1 1899542 131848 70c8d3a4 1899542