diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..af5a6d1a1c8cd34bbbdaed9698c2d09a78621abb --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,44 @@ +FROM centos:7.8.2003 +RUN yum update -y && yum install -y \ +epel-release \ +cmake \ +libtool \ +automake \ +gcc \ +gcc-c++ \ +redhat-lsb \ +libuuid-devel \ +libaio-devel \ +flex \ +&& yum install -y \ +bison \ +jemalloc-devel \ +libtirpc-devel \ +nfs-utils \ +libattr \ +libattr-devel \ +etcd \ +redis \ +curl-devel \ +&& yum install -y \ +yajl-devel \ +hiredis-devel \ +python-paramiko \ +expect \ +gperftools \ +sqlite-devel \ +fuse-devel \ +openssl-devel \ +rpcbind \ +yasm \ +python2-pip \ +git-tools \ +openssh-server && yum clean all + +RUN pip install futurist futures && pip install -Iv dnspython==1.16.0 python-etcd==0.4.5 && rm -rf ~/.cache/pip/* +RUN git clone https://github.com/01org/isa-l.git && cd isa-l && ./autogen.sh && ./configure && make install && cd ../ && rm -rf isa-l +RUN git clone https://toscode.gitee.com/FusionStack/SimpleDFS.git && cd ./SimpleDFS && mkdir build && cd build && cmake .. && make && make install && cd .. +RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key && ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key && ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key +RUN perl -i -nlpe 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config +RUN echo 'root:sdfs_password' | chpasswd +CMD ["/usr/sbin/sshd", "-D"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000000000000000000000000000000000..508d7e8bdff356b083ec271f61d37470bcce0c86 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,130 @@ +# SimpleDFS docker版本安装运行 + + + +## 一、构建SimpleDFS镜像 + +运行以下命令 + +``` +build.sh +``` + + + +构建成功会在本地生成镜像，目前版本为0.0.2 + +``` +REPOSITORY TAG IMAGE ID CREATED SIZE +sdfs 0.0.2 f86129ac8705 41 minutes ago 937MB + +``` + + + +## 二、运行SimpleDFS集群容器 + +``` +docker-compose up -d +``` + + + + + +## 三、启动服务 + + + +如果是第一次运行以下两个命令创建集群 + +``` +/opt/sdfs/app/admin/cluster.py sshkey --hosts node1,node2,node3 + +``` + +这个命令需要用到ssh密码，密码为：sdfs_password + + + +创建集群 + +``` +/opt/sdfs/app/admin/cluster.py create --hosts node1,node2,node3 + +``` + + + +## 四、运行测试 + +进入某节点所在容器 + +``` +docker exec -it sdfs_node1_1 /bin/bash +``` + +运行： + +``` +sdfs mkdir /test/ +``` + + + +查看创建结果： + +``` +sdfs ls / +``` + + + +#### 1、创建并挂载NFS + + + +进入某节点所在容器后，运行以下命令 + +``` +sdfs share -s -m rw -n test -p nfs -H node1,node2,node3 /test +``` + + + +成功后，运行挂载nfs到该机器 + +``` + +mkdir /mnt/test +mount -t nfs -o async,soft,rw,intr,nolock,tcp,nfsvers=3 `uname -n`:/test /mnt/test +``` + + + +#### 2、用程序进行测试 + +首先，拷贝测试程序目录到容器内， + +在宿主机内运行以下命令，执行拷贝 + +``` +docker cp write_test/ <节点容器id>:/ +``` + + + +运行测试程序 + +``` +cd /write_test + +make + + +./write_test -t 16 -f 800 -s 1 -l 16 -c 128 -p /mnt/test/5w + + +/write_test -t 32 -f 800 -s 1 -l 16 -c 128 -p /mnt/test/10w +``` + diff --git a/docker/build.sh b/docker/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..822e11b157297f6cc98f5afbd16b687c8f797dfa --- /dev/null +++ b/docker/build.sh @@ -0,0 +1 @@ +docker build -t sdfs:0.0.3 . diff --git a/docker/cluster.conf b/docker/cluster.conf new file mode 100644 index 0000000000000000000000000000000000000000..7fbc0058c000f8978469c91d22ff4152c33476f1 --- /dev/null +++ b/docker/cluster.conf @@ -0,0 +1,3 @@ +node1 cds[0] mond[0] redis[0] nfs[0] +node3 cds[0] mond[0] redis[0] nfs[0] +node2 cds[0] mond[0] redis[0] nfs[0] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..567a510000a26d94f2b2b202373250d974999b29 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,86 @@ +version: '3' +services: + node1: + image: sdfs:0.0.3 + hostname: node1 + privileged: true + volumes: + - ./cluster.conf:/opt/sdfs/etc/cluster.conf + - ./sdfs.conf:/opt/sdfs/etc/sdfs.conf + - /proc/sys:/writable-sys + - /opt/sdfs/data/cds/0 + - /opt/sdfs/data/redis/0 + sysctls: + net.ipv4.ip_forward: 0 + #net.core.wmem_max: 104857600 + #net.core.rmem_max: 104857600 + ulimits: + nproc: -1 + nofile: + soft: 100000 + hard: 100000 + memlock: + soft: -1 + hard: -1 + networks: + main: + aliases: + - node1 + command: ['/usr/sbin/init'] + node2: + image: sdfs:0.0.3 + hostname: node2 + privileged: true + volumes: + - ./cluster.conf:/opt/sdfs/etc/cluster.conf + - ./sdfs.conf:/opt/sdfs/etc/sdfs.conf + - /proc/sys:/writable-sys + - /opt/sdfs/data/cds/0 + - /opt/sdfs/data/redis/0 + sysctls: + net.ipv4.ip_forward: 0 + #net.core.wmem_max: 104857600 + #net.core.rmem_max: 104857600 + ulimits: + nproc: -1 + nofile: + soft: 100000 + hard: 100000 + memlock: + soft: -1 + hard: -1 + networks: + main: + aliases: + - node2 + command: ['/usr/sbin/init'] + node3: + image: sdfs:0.0.3 + hostname: node3 + privileged: true + volumes: + - ./cluster.conf:/opt/sdfs/etc/cluster.conf + - ./sdfs.conf:/opt/sdfs/etc/sdfs.conf + - /proc/sys:/writable-sys + - /opt/sdfs/data/cds/0 + - /opt/sdfs/data/redis/0 + sysctls: + net.ipv4.ip_forward: 0 + #net.core.wmem_max: 104857600 + #net.core.rmem_max: 104857600 + ulimits: + nproc: -1 + nofile: + soft: 100000 + hard: 100000 + memlock: + soft: -1 + hard: -1 + networks: + main: + aliases: + - node3 + command: ['/usr/sbin/init'] + +networks: + main: diff --git a/docker/sdfs.conf b/docker/sdfs.conf new file mode 100644 index 0000000000000000000000000000000000000000..2b3a6d8fc78f073db6246818b5a768f53aff3e0f --- /dev/null +++ b/docker/sdfs.conf @@ -0,0 +1,23 @@ +globals { + chunk_rep 2; + rpc_timeout 20; + performance_analysis 1; + redis_sharding 8; + redis_replica 1; + check_mountpoint off; + polling_timeout 0; + check_version off; + #testing off; + backtrace off; + aio_thread off; + home /opt/sdfs/data; + #maxcore on; + #nfs ganesha; + networks { + 172.19.0.0/24; + } +} +mds { + disk_keep 10G; +} + diff --git a/docker/write_test/Makefile b/docker/write_test/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..7895ebfe1ce345feb79558663e63e4641617cf43 --- /dev/null +++ b/docker/write_test/Makefile @@ -0,0 +1,123 @@ +############################################################################### +# +# Generic Makefile for C/C++ Program +# +# Author: whyglinux (whyglinux AT hotmail DOT com) +# Date: 2006/03/04 +# Description: +# The makefile searches in directories for the source files +# with extensions specified in , then compiles the sources +# and finally produces the , the executable file, by linking +# the objectives. +# Usage: +# $ make compile and link the program. +# $ make objs compile only (no linking. Rarely used). +# $ make clean clean the objectives and dependencies. +# $ make cleanall clean the objectives, dependencies and executable. +# $ make rebuild rebuild the program. The same as make clean && make all. +#============================================================================== +## Customizing Section: adjust the following if necessary. +##============================================================================= +# The executable file name. +# It must be specified. +# PROGRAM := a.out # the executable name +PROGRAM := write_test +# The directories in which source files reside. +# At least one path should be specified. +# SRCDIRS := . # current directory +SRCDIRS := . +# The source file types (headers excluded). +# At least one type should be specified. +# The valid suffixes are among of .c, .C, .cc, .cpp, .CPP, .c++, .cp, or .cxx. +# SRCEXTS := .c # C program +# SRCEXTS := .cpp # C++ program +# SRCEXTS := .c .cpp # C/C++ program +SRCEXTS := .c +# The flags used by the cpp (man cpp for more). +# CPPFLAGS := -Wall -Werror # show all warnings and take them as errors +CPPFLAGS := +# The compiling flags used only for C. +# If it is a C++ program, no need to set these flags. +# If it is a C and C++ merging program, set these flags for the C parts. +CFLAGS := -Wall -O2 -g +CFLAGS += +# The compiling flags used only for C++. +# If it is a C program, no need to set these flags. +# If it is a C and C++ merging program, set these flags for the C++ parts. +CXXFLAGS := +CXXFLAGS += +# The library and the link options ( C and C++ common). +LDFLAGS := -lpthread +LDFLAGS += +## Implict Section: change the following only when necessary. +##============================================================================= +# The C program compiler. Uncomment it to specify yours explicitly. +#CC = gcc +# The C++ program compiler. Uncomment it to specify yours explicitly. +#CXX = g++ +# Uncomment the 2 lines to compile C programs as C++ ones. +#CC = $(CXX) +#CFLAGS = $(CXXFLAGS) +# The command used to delete file. +#RM = rm -f +## Stable Section: usually no need to be changed. But you can add more. +##============================================================================= +SHELL = /bin/sh +SOURCES = $(foreach d,$(SRCDIRS),$(wildcard $(addprefix $(d)/*,$(SRCEXTS)))) +OBJS = $(foreach x,$(SRCEXTS),$(patsubst %$(x),%.o,$(filter %$(x),$(SOURCES)))) +DEPS = $(patsubst %.o,%.d,$(OBJS)) +.PHONY : all objs clean cleanall rebuild +all : $(PROGRAM) +# Rules for creating the dependency files (.d). +#--------------------------------------------------- +%.d : %.c + @$(CC) -MM -MD $(CFLAGS) $< +%.d : %.C + @$(CC) -MM -MD $(CXXFLAGS) $< +%.d : %.cc + @$(CC) -MM -MD $(CXXFLAGS) $< +%.d : %.cpp + @$(CC) -MM -MD $(CXXFLAGS) $< +%.d : %.CPP + @$(CC) -MM -MD $(CXXFLAGS) $< +%.d : %.c++ + @$(CC) -MM -MD $(CXXFLAGS) $< +%.d : %.cp + @$(CC) -MM -MD $(CXXFLAGS) $< +%.d : %.cxx + @$(CC) -MM -MD $(CXXFLAGS) $< +# Rules for producing the objects. +#--------------------------------------------------- +objs : $(OBJS) +%.o : %.c + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< +%.o : %.C + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< +%.o : %.cc + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< +%.o : %.cpp + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< +%.o : %.CPP + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< +%.o : %.c++ + $(CXX -c $(CPPFLAGS) $(CXXFLAGS) $< +%.o : %.cp + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< +%.o : %.cxx + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< +# Rules for producing the executable. +#---------------------------------------------- +$(PROGRAM) : $(OBJS) +ifeq ($(strip $(SRCEXTS)), .c) # C file + $(CC) -o $(PROGRAM) $(OBJS) $(LDFLAGS) +else # C++ file + $(CXX) -o $(PROGRAM) $(OBJS) $(LDFLAGS) +endif +-include $(DEPS) +rebuild: clean all +clean : + @$(RM) *.o *.d +cleanall: clean + @$(RM) $(PROGRAM) $(PROGRAM).exe +### End of the Makefile ## Suggestions are welcome ## All rights reserved ### +############################################################################### \ No newline at end of file diff --git a/docker/write_test/write_test.c b/docker/write_test/write_test.c new file mode 100644 index 0000000000000000000000000000000000000000..cef932315ee5af1ddd84bd6b233e6c654064bd7c --- /dev/null +++ b/docker/write_test/write_test.c @@ -0,0 +1,231 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define _DATETIME_SIZE 32 +#define PATH_LEN 256 + + +struct write_args { + char root_dir[PATH_LEN]; + int file_num; + int min_size; + int max_size; + int write_num; +}; + +struct option long_options[] = +{ + {"path", 1, NULL, 'p'}, + {"thread", 1, NULL, 't'}, + {"file", 1, NULL, 'f'}, + {"min_size", 1, NULL, 's'}, + {"max_size", 1, NULL, 'l'}, + {"write_times", 1, NULL, 'c'}, + {"help", 0, NULL, 'h'}, + {0, 0, 0, 0}, +}; + +static void dump_args(struct write_args *args) +{ + fprintf(stderr, "root_dir:\t %s\n", args->root_dir); + fprintf(stderr, "file_num:\t %d\n", args->file_num); + fprintf(stderr, "min_size:\t %d\n", args->min_size); + fprintf(stderr, "max_size:\t %d\n", args->max_size); + fprintf(stderr, "write_times:\t %d\n", args->write_num); +} + +/** + * Get current date, return in 'YYYYMMDD' format + * + */ +static int get_date(char *date) +{ + time_t seconds; + struct tm *p_tm; + + time(&seconds); + p_tm = localtime(&seconds); + + sprintf(date, "%04d%02d%02d", + p_tm->tm_year + 1900, p_tm->tm_mon + 1, p_tm->tm_mday); + + return 0; +} + +/** + * write random length file to BASE_DIR/`date` + */ +void *writer(void *arg) +{ + struct write_args *args = (struct write_args *)arg; + int fd[args->file_num]; + int i, j; + char full_path[PATH_LEN] = {'\0'}; + struct timeval tv; + + pthread_t tid = pthread_self(); + + // open files: + for (i = 0; i < args->file_num; i++) + { + // generate file name + //get_time(create_time); + gettimeofday(&tv, NULL); + + snprintf(full_path, 1024, "%s/%ld_%ld_%06d.data", + args->root_dir, tv.tv_sec, tv.tv_usec, i); + + fd[i] = open(full_path, O_WRONLY | O_CREAT, 0755); + if (fd[i] == -1) + { + fprintf(stderr, "open file %d of thread %lu failed!\n", + i, tid); + } + /*else + { + fprintf(stdout, "file %s is open, fd is %d\n", full_path, fd[i]); + } */ + + } + + fprintf(stdout, "all files are open\n"); + time_t t; + int write_len; + // write random data with random size + for (i = 0; i < args->write_num; i++) + { + srand((unsigned) time(&t)); + write_len = rand() % (args->max_size - args->min_size) + args->min_size; + char *buffer = (char *) malloc(write_len); // just use malloc's garbage data + for (j=0; j< args->file_num; j++) + { + if (write(fd[j], buffer, write_len) != write_len) + { + fprintf(stderr, "write to fd[%d] size is not equal to write_len\n!", i); + } + } + free(buffer); + + } + + // close all files + for (i = 0; i < args->file_num; i++) + { + close(fd[i]); + } + + return (void *)0; +} + +static void usage() +{ + const char *USAGE = "" \ + "./write_test [options]\n" \ + "options are:\n" \ + " -p --path the root path for write\n" \ + " -t --thread number of threads to run\n" \ + " -f --file number file to generate for per thread\n" \ + " -s --min_size the minimal write size for per write call, size in KB\n" \ + " -l --max_size the maxmum write size for per write call, size in KB\n" \ + " -c --write_times write times\n" \ + " -h --help show this help\n"; + + fprintf(stderr, "%s", USAGE); +} + + +int main(int argc, char **argv) { + char * short_options = "p:t:f:s:l:c:h"; + char date[9] = {'\0'}; + int c; + int thread_num = 0; + + struct write_args *args = (struct write_args *)malloc(sizeof(struct write_args)); + memset(args, 0, sizeof(struct write_args)); + + while ((c = getopt_long(argc, argv, short_options, long_options, NULL)) != -1) + { + switch (c) + { + case 'p': + fprintf(stdout, "args: %s\n", optarg); + strncpy(args->root_dir, optarg, PATH_LEN - 1); + break; + case 't': + thread_num = atoi(optarg); + break; + case 'f': + args->file_num = atoi(optarg); + break; + case 's': + args->min_size = atoi(optarg); + args->min_size = args->min_size * 1024; + break; + case 'l': + args->max_size = atoi(optarg); + args->max_size = args->max_size * 1024; + break; + case 'c': + args->write_num = atoi(optarg); + break; + case 'h': + default: + usage(); + break; + } + } + + pthread_t write_thread[thread_num]; + int i; + + get_date(date); + //snprintf(args->root_dir, sizeof(args->root_dir), "%s%s", args->, date); + fprintf(stdout, "root_dir: %s\n", args->root_dir); + + if (access(args->root_dir, R_OK) != 0) + { + fprintf(stderr, "Dest dir doesn't exist, create it..\n"); + if(mkdir(args->root_dir, 0755) != 0) + { + perror("mkdir"); + free(args); + return -1; + } + } + + dump_args(args); + + for (i=0; i < thread_num; i++) + { + if (pthread_create(&write_thread[i], NULL, writer, (void *)args)) + { + fprintf(stderr, "create thread[%d] failed!\n", i); + break; + } + } + + for (i = 0; i < thread_num; i++) + { + if (pthread_join(write_thread[i], NULL)) + { + fprintf(stderr, "wait for thread[%d] failed!\n", i); + break; + } + } + + free(args); + return 0; +} + + diff --git a/docker/write_test/write_test.d b/docker/write_test/write_test.d new file mode 100644 index 0000000000000000000000000000000000000000..d97af89780653e47077276d9ba6baa946c828b41 --- /dev/null +++ b/docker/write_test/write_test.d @@ -0,0 +1 @@ +write_test.o: write_test.c