From 703ebf09d55e95289ac98937966ea2e7452d2a03 Mon Sep 17 00:00:00 2001 From: WenJieLi Date: Mon, 30 May 2022 09:37:43 +0800 Subject: [PATCH] update project from code.gstore.cn --- .debug/.gitignore | 6 +- .gitattributes | 108 +- Database/Database.cpp | 32 +- Dockerfile | 30 +- KVstore/KVstore.cpp | 3 +- Main/gexport.cpp | 2 +- Main/ghttp.cpp | 1692 +++++++++++++------- Main/ginit.cpp | 2 + Main/gmonitor.cpp | 2 +- Main/grestore.cpp | 2 +- Parser/QueryParser.cpp | 4 +- Query/GeneralEvaluation.cpp | 28 +- README.md | 158 +- Util/INIParser.cpp | 5 +- Util/IPBlackList.cpp | 35 + Util/IPBlackList.h | 12 +- Util/IPWhiteList.cpp | 35 + Util/IPWhiteList.h | 10 + Util/Latch.cpp | 8 + Util/Util.cpp | 234 ++- Util/Util.h | 5 +- backup.json | 1 + conf.ini | 13 +- docs/API.md | 2881 +++++++---------------------------- docs/DEMAND.md | 53 +- docs/DOCKER_DEPLOY_CN.md | 15 +- docs/DOCKER_DEPLOY_EN.md | 22 +- makefile | 3 +- 28 files changed, 2254 insertions(+), 3147 deletions(-) diff --git a/.debug/.gitignore b/.debug/.gitignore index fa701604..a5baada1 100644 --- a/.debug/.gitignore +++ b/.debug/.gitignore @@ -1,3 +1,3 @@ -* -!.gitignore - +* +!.gitignore + diff --git a/.gitattributes b/.gitattributes index b7eb9d2a..81b9a0a7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,54 +1,54 @@ -# https://git-scm.com/docs/gitattributes -# We find that using this file may cause problem between different branches -# Therefore, all are commented. - -# Set the default behavior, in case people don't have core.autocrlf set. -#* text=auto -#*.txt text -#*.vcproj text eol=crlf -#*.sh text eol=lf -#*.py eol=lf -#*.c text eol=lf -#*.h text eol=lf -#*.cpp text eol=lf -#*.cu text eol=lf -#*.md text eol=lf -#*.nt text eol=lf -#*.sql text eol=lf -#*.jpg -text - -# Explicitly declare text files you want to always be normalized and converted -# to native line endings on checkout. -#*.c text -#*.h text -#*.md text -#*.js text -#*.json text -#*.wxss text -#*.wxml text - -# Declare files that will always have CRLF line endings on checkout. -#*.sln text eol=crlf - -# Denote all files that are truly binary and should not be modified. -#*.png binary -#*.jpg binary - -# Below denotes which file to use git-lfs, large files -#*.zip filter=lfs diff=lfs merge=lfs -text -#*.tar.gz filter=lfs diff=lfs merge=lfs -text -#*.7z filter=lfs diff=lfs merge=lfs -text -#*.rar filter=lfs diff=lfs merge=lfs -text -#*.torrent filter=lfs diff=lfs merge=lfs -text -#*.iso filter=lfs diff=lfs merge=lfs -text -#*.jpg filter=lfs diff=lfs merge=lfs -text -#*.jpeg filter=lfs diff=lfs merge=lfs -text -#*.png filter=lfs diff=lfs merge=lfs -text -#*.pdf filter=lfs diff=lfs merge=lfs -text -#*.mp3 filter=lfs diff=lfs merge=lfs -text -#*.wav filter=lfs diff=lfs merge=lfs -text -#*.mp4 filter=lfs diff=lfs merge=lfs -text -#*.rmvb filter=lfs diff=lfs merge=lfs -text -#*.nt filter=lfs diff=lfs merge=lfs -text -#*.n3 filter=lfs diff=lfs merge=lfs -text - +# https://git-scm.com/docs/gitattributes +# We find that using this file may cause problem between different branches +# Therefore, all are commented. + +# Set the default behavior, in case people don't have core.autocrlf set. +#* text=auto +#*.txt text +#*.vcproj text eol=crlf +#*.sh text eol=lf +#*.py eol=lf +#*.c text eol=lf +#*.h text eol=lf +#*.cpp text eol=lf +#*.cu text eol=lf +#*.md text eol=lf +#*.nt text eol=lf +#*.sql text eol=lf +#*.jpg -text + +# Explicitly declare text files you want to always be normalized and converted +# to native line endings on checkout. +#*.c text +#*.h text +#*.md text +#*.js text +#*.json text +#*.wxss text +#*.wxml text + +# Declare files that will always have CRLF line endings on checkout. +#*.sln text eol=crlf + +# Denote all files that are truly binary and should not be modified. +#*.png binary +#*.jpg binary + +# Below denotes which file to use git-lfs, large files +#*.zip filter=lfs diff=lfs merge=lfs -text +#*.tar.gz filter=lfs diff=lfs merge=lfs -text +#*.7z filter=lfs diff=lfs merge=lfs -text +#*.rar filter=lfs diff=lfs merge=lfs -text +#*.torrent filter=lfs diff=lfs merge=lfs -text +#*.iso filter=lfs diff=lfs merge=lfs -text +#*.jpg filter=lfs diff=lfs merge=lfs -text +#*.jpeg filter=lfs diff=lfs merge=lfs -text +#*.png filter=lfs diff=lfs merge=lfs -text +#*.pdf filter=lfs diff=lfs merge=lfs -text +#*.mp3 filter=lfs diff=lfs merge=lfs -text +#*.wav filter=lfs diff=lfs merge=lfs -text +#*.mp4 filter=lfs diff=lfs merge=lfs -text +#*.rmvb filter=lfs diff=lfs merge=lfs -text +#*.nt filter=lfs diff=lfs merge=lfs -text +#*.n3 filter=lfs diff=lfs merge=lfs -text + diff --git a/Database/Database.cpp b/Database/Database.cpp index 7685db13..6bd66661 100644 --- a/Database/Database.cpp +++ b/Database/Database.cpp @@ -22,6 +22,7 @@ Database::Database() this->id_tuples_file = "id_tuples"; this->update_log = "update.log"; this->update_log_since_backup = "update_since_backup.log"; + this->csr = NULL; // this->csr = new CSR[2]; string kv_store_path = store_path + "/kv_store"; @@ -79,6 +80,7 @@ Database::Database(string _name) this->id_tuples_file = "id_tuples"; this->update_log = "update.log"; this->update_log_since_backup = "update_since_backup.log"; + this->csr = NULL; // this->csr = new CSR[2]; string kv_store_path = store_path + "/kv_store"; @@ -5187,6 +5189,7 @@ Database::batch_insert(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _tripl unsigned update_num_s = 0; unsigned update_num_p = 0; unsigned update_num_o = 0; + unsigned update_num_subject=0; if (!_is_restore) { write_update_log(_triples, _triple_num, 1, txn); @@ -5200,6 +5203,7 @@ Database::batch_insert(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _tripl if (_sub_id == INVALID_ENTITY_LITERAL_ID) { _sub_id = this->allocEntityID(); + update_num_subject=update_num_subject+1; (this->kvstore)->setIDByEntity(_triple.subject, _sub_id); (this->kvstore)->setEntityByID(_sub_id, _triple.subject); vertices.push_back(_sub_id); @@ -5286,11 +5290,21 @@ Database::batch_insert(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _tripl pre_t.join(); obj_t.join(); + unsigned update_num_triple=update_num_s; + if(update_num_triplestringindex->SetTrie(kvstore->getTrie()); + this->triples_num=this->triples_num+update_num_triple; + this->sub_num=this->sub_num+update_num_subject; //update string index this->stringindex->change(vertices, *this->kvstore, true); this->stringindex->change(predicates, *this->kvstore, false); @@ -5307,6 +5321,8 @@ Database::batch_remove(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _tripl unsigned update_num_s = 0; unsigned update_num_p = 0; unsigned update_num_o = 0; + unsigned update_num_subject=0; + unsigned update_num_triple=0; vector vertices, predicates; set sub_ids, pre_ids, obj_ids; @@ -5367,9 +5383,16 @@ Database::batch_remove(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _tripl pre_t.join(); obj_t.join(); //cout << update_num_s << " " << update_num_p << " " << update_num_o << endl; - assert(update_num_o == update_num_p); - assert(update_num_s == update_num_o); + // assert(update_num_o == update_num_p); + // assert(update_num_s == update_num_o); + if(update_num_triplestringindex->SetTrie(kvstore->getTrie()); //update string index + this->triples_num=this->triples_num-update_num_triple; this->stringindex->disable(vertices, true); this->stringindex->disable(predicates, false); } diff --git a/Dockerfile b/Dockerfile index f66682fa..65bb8ac4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,25 +4,27 @@ #It is best to update the process of modification in the rear of the original code(reduce the order changes). #Otherwise the speed of construction will be influenced on a large scale. -FROM lsvih/gcc-boost-cmake-java:v1 -RUN apt update \ - && apt install -y --no-install-recommends \ - libcurl4-openssl-dev \ - libreadline-dev \ - uuid-dev \ - && ldconfig -v \ - && echo "* - nofile 65535" >> /etc/security/limits.conf \ - && echo "* - noproc 65535" >> /etc/security/limits.conf +#TODO: some space can be saved by using the low version of gcc mirror(e.g-gcc:5). +#But its mobility and dependence has not been tested yet and waiting for confirmation. + +FROM registry.docker-cn.com/library/gcc:8 + +#download all the optional installation library in gstore's document. +RUN apt-get update && apt-get install -y --no-install-recommends realpath \ + ccache \ + vim \ + lsof \ + openjdk-8-jdk \ + libreadline-dev \ + libboost-all-dev \ + && rm -rf /var/lib/apt/lists/ COPY . /usr/src/gstore WORKDIR /usr/src/gstore +#The solution to the problem of java, whose default setting is using ansii to encode. ENV LANG C.UTF-8 EXPOSE 80 -RUN make pre -j && make -j \ - && apt autoclean && apt clean \ - && rm -rf /tmp/* /var/tmp/* \ - && rm -rf /usr/share/doc/* \ - && rm -rf /var/lib/apt/lists/* +RUN make diff --git a/KVstore/KVstore.cpp b/KVstore/KVstore.cpp index 51b8e365..d66f8a6a 100644 --- a/KVstore/KVstore.cpp +++ b/KVstore/KVstore.cpp @@ -5778,12 +5778,11 @@ KVstore::o2values_literal_vacuum(vector& obj_literal_ids, shared_ptrclean_dirty_key(this->objID2values, _obj_id); + this->clean_dirty_key(this->objID2values_literal, _obj_id - Util::LITERAL_FIRST_ID); if(_len == 0) { if(base_empty == false) { cerr << "delete keys" << endl; - assert(false); this->removeKey(this->objID2values, _obj_id); } continue; diff --git a/Main/gexport.cpp b/Main/gexport.cpp index ce6a6c9f..ed444ec6 100644 --- a/Main/gexport.cpp +++ b/Main/gexport.cpp @@ -85,7 +85,7 @@ main(int argc, char * argv[]) ResultSet ask_rs; FILE* ask_ofp = stdout; int ret = system_db.query(sparql, ask_rs, ask_ofp); - if (ask_rs.answer[0][0] == "false") + if (ask_rs.answer[0][0] == "\"false\"^^") { cout<<"The database does not exist."< HttpClient; #define SYSTEM_PATH "data/system/system.nt" #define MAX_QUERYLOG_size 800000000 #define QUERYLOG_PATH "logs/endpoint/" +#define IP_ACCESS_PATH "logs/ipaccess/" #define SYSTEM_USERNAME "system" #define MAX_OUTPUT_SIZE 100000 #define TEST_IP "106.13.13.193" @@ -91,6 +92,8 @@ bool db_checkpoint(string db_name); txn_id_t get_txn_id(string db_name, string user); +void writeIpAccessLog(string ip,string operation,string msg,int code); + //bool doQuery(string format, string db_query, const HttpServer& server, const shared_ptr& response, const shared_ptr& request); //============================================================================= @@ -112,75 +115,84 @@ bool checkdbexist(string db_name); bool checkdbload(string db_name); +bool checkIp(string ip); + +void addErrorNumForIp(string ip); + string getRemoteIp(const shared_ptr& request); bool trylockdb(std::map::iterator it_already_build); -void writeLog(FILE* fp, string _info); +void writeLog(string _info); -void sendResponseMsg(int code, string msg, const shared_ptr& response); +void sendResponseMsg(int code, string msg, const shared_ptr& response,string ip,string operation); -void build_thread_new(const shared_ptr& response, string db_name, string db_path, string username, string password); +void build_thread_new(const shared_ptr& response, string db_name, string db_path, string username, string password,string ip); -void load_thread_new(const shared_ptr& response, string db_name); +void load_thread_new(const shared_ptr& response, string db_name,string ip, bool load_csr); -void monitor_thread_new(const shared_ptr& response, string db_name); +void monitor_thread_new(const shared_ptr& response, string db_name,string ip); -void unload_thread_new(const shared_ptr& response, string db_name); +void unload_thread_new(const shared_ptr& response, string db_name,string ip); -void drop_thread_new(const shared_ptr& response, string db_name,string is_backup); +void drop_thread_new(const shared_ptr& response, string db_name,string is_backup,string ip); -void show_thread_new(const shared_ptr& response); +void show_thread_new(const shared_ptr& response,string username,string ip); -void userManager_thread_new(const shared_ptr& response,string username,string password,string type); +void userManager_thread_new(const shared_ptr& response,string username,string password,string type,string ip); -void showuser_thread_new(const shared_ptr& response); +void showuser_thread_new(const shared_ptr& response,string ip); void userPrivilegeManage_thread_new(const shared_ptr& response,string username,string privilege, -string type,string db_name); +string type,string db_name,string ip); -void backup_thread_new(const shared_ptr& response,string db_name,string backup_path); +void backup_thread_new(const shared_ptr& response,string db_name,string backup_path,string ip); -void restore_thread_new(const shared_ptr& response,string db_name,string backup_path,string username); +void restore_thread_new(const shared_ptr& response,string db_name,string backup_path,string username,string ip); void query_thread_new(const shared_ptr& response,string db_name,string sparql,string format, string update_flag,string remote_ip,string log_prefix); -void export_thread_new(const shared_ptr& response,string db_name,string db_path,string username); +void export_thread_new(const shared_ptr& response,string db_name,string db_path,string username,string ip); -void login_thread_new(const shared_ptr& response); +void login_thread_new(const shared_ptr& response,string ip); -void check_thread_new(const shared_ptr& response); +void check_thread_new(const shared_ptr& response,string ip); -void begin_thread_new(const shared_ptr& response,string db_name,string isolevel,string username); +void begin_thread_new(const shared_ptr& response,string db_name,string isolevel,string username,string ip); -void tquery_thread_new(const shared_ptr& response,string db_name,string TID_s,string sparql); +void tquery_thread_new(const shared_ptr& response,string db_name,string TID_s,string sparql,string ip); -void commit_thread_new(const shared_ptr& response,string db_name,string TID_s); +void commit_thread_new(const shared_ptr& response,string db_name,string TID_s,string ip); -void rollback_thread_new(const shared_ptr& response,string db_name,string TID_s); +void rollback_thread_new(const shared_ptr& response,string db_name,string TID_s,string ip); -void txnlog_thread_new(const shared_ptr& response,string username); +void txnlog_thread_new(const shared_ptr& response,string username, int page_no, int page_size, string ip); -void checkpoint_thread_new(const shared_ptr& response,string db_name); +void checkpoint_thread_new(const shared_ptr& response,string db_name,string ip); -void test_connect_thread_new(const shared_ptr& response); +void test_connect_thread_new(const shared_ptr& response,string ip); -void getCoreVersion_thread_new(const shared_ptr& response); +void getCoreVersion_thread_new(const shared_ptr& response,string ip); -void batchInsert_thread_new(const shared_ptr& response,string db_name,string file); +void batchInsert_thread_new(const shared_ptr& response,string db_name,string file,string ip); -void batchRemove_thread_new(const shared_ptr& response,string db_name,string file); +void batchRemove_thread_new(const shared_ptr& response,string db_name,string file,string ip); bool checkall_thread(const shared_ptr& response, const shared_ptr& request); ifstream & seek_to_line(ifstream &, int); -void quereyLog_thread_new(const shared_ptr& response, string file_name, int page_no, int page_size); +void quereyLog_thread_new(const shared_ptr& response, string file_name, int page_no, int page_size,string ip); + +void accessLog_thread_new(const shared_ptr& response, string file_name, int page_no, int page_size,string ip); + +void IPManager_thread_new(const shared_ptr& response, string ips, string ip_type, string type); //TODO: use lock to protect logs when running in multithreading environment FILE* query_logfp = NULL; string queryLog = "logs/endpoint/query.log"; mutex query_log_lock; +mutex ip_log_lock; string CoreVersion; string APIVersion; string system_password; @@ -197,6 +209,7 @@ IPBlackList* ipBlackList; pthread_rwlock_t databases_map_lock; pthread_rwlock_t already_build_map_lock; pthread_rwlock_t users_map_lock; +pthread_rwlock_t ips_map_lock; pthread_rwlock_t txn_m_lock; Database *system_database; @@ -205,6 +218,57 @@ std::map databases; std::map> txn_managers; std::map running_txn; //database information + +struct IpInfo{ + private: + int accessNum=0; + int successNum=0; + int errorNum=0; + std::string ip; + public: + IpInfo(){ + + } + IpInfo(string name) + { + ip=name; + accessNum=0; + successNum=0; + errorNum=0; + } + int getAccessNum() + { + return accessNum; + } + int getSuccessNum() + { + return successNum; + } + int getErrorNum() + { + return errorNum; + } + string getName() + { + return ip; + } + void setAccessNum(int value) + { + accessNum=value; + } + void setSuccessNum(int value) + { + successNum=value; + } + void setErrorNum(int value) + { + errorNum=value; + } + void setName(string value) + { + ip=value; + } +}; struct DBInfo{ private: std::string db_name;//! the name of database @@ -435,6 +499,7 @@ struct User{ //struct User root = User(ROOT_USERNAME, ROOT_PASSWORD); std::map already_build; std::map users; +std::map ips; //struct User root = User(ROOT_USERNAME, ROOT_PASSWORD); //users.insert(pair(ROOT_USERNAME, &root)); //users[ROOT_USERNAME] = &root; @@ -853,17 +918,16 @@ int initialize(int argc, char *argv[]) if (argc < 2) { - /*cout << "please input the complete command:\t" << endl; - cout << "\t bin/gadd -h" << endl;*/ - cout << "Use the default port:9000!" << endl; - cout << "Not load any database!" << endl; - server.config.port = 9000; - port = server.config.port; - db_name = ""; - loadCSR = 0; - whiteList = 0; - blackList = 0; - + /*cout << "please input the complete command:\t" << endl; + cout << "\t bin/gadd -h" << endl;*/ + cout << "Use the default port:9000!" << endl; + cout << "Not load any database!" << endl; + server.config.port = 9000; + port = server.config.port; + db_name = ""; + loadCSR = 0; + whiteList = 0; + blackList = 0; } else if (argc == 2) { @@ -911,6 +975,8 @@ int initialize(int argc, char *argv[]) loadCSR = Util::string2int(Util::getArgValue(argc, argv, "c", "csr", "0")); ipWhiteFile = Util::getConfigureValue("ip_allow_path"); ipBlackFile = Util::getConfigureValue("ip_deny_path"); + cout<<"ipWhiteFile:"<& response,string db_name,string db_path,string username,string password) +void build_thread_new(const shared_ptr& response,string db_name,string db_path,string username,string password,string ip) { + string operation="build"; if (db_path == SYSTEM_PATH) { string error = "You have no rights to access system files"; - sendResponseMsg(1002, error, response); + sendResponseMsg(1002, error, response,ip,operation); return; } string result = checkparamValue("db_name", db_name); if (result.empty() == false) { - sendResponseMsg(1003, result, response); + sendResponseMsg(1003, result, response,ip,operation); return; } @@ -1340,7 +1407,7 @@ void build_thread_new(const shared_ptr& response,string db if (checkdbexist(db_name)) { string error = "database already built."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } string dataset = db_path; @@ -1357,7 +1424,7 @@ void build_thread_new(const shared_ptr& response,string db string error = "Import RDF file to database failed."; string cmd = "rm -r " + database + ".db"; system(cmd.c_str()); - sendResponseMsg(1005, error, response); + sendResponseMsg(1005, error, response,ip,operation); return; } @@ -1369,7 +1436,7 @@ void build_thread_new(const shared_ptr& response,string db if (addPrivilege(username, "query", db_name) == 0 || addPrivilege(username, "load", db_name) == 0 || addPrivilege(username, "unload", db_name) == 0 || addPrivilege(username, "backup", db_name) == 0 || addPrivilege(username, "restore", db_name) == 0 || addPrivilege(username, "export", db_name) == 0) { string error = "add query or load or unload privilege failed."; - sendResponseMsg(1006, error, response); + sendResponseMsg(1006, error, response,ip,operation); return; } cout << "add query and load and unload privilege succeed after build." << endl; @@ -1387,7 +1454,7 @@ void build_thread_new(const shared_ptr& response,string db cout << "database add done." << endl; // string success = db_name + " " + db_path; string success = "Import RDF file to database done."; - sendResponseMsg(0, success, response); + sendResponseMsg(0, success, response,ip,operation); Util::add_backuplog(db_name); } @@ -1398,12 +1465,13 @@ void build_thread_new(const shared_ptr& response,string db * @param {string} msg:StatusMsg value * @param {const} shared_ptr */ -void sendResponseMsg(int code, string msg, const shared_ptr& response) +void sendResponseMsg(int code, string msg, const shared_ptr& response,string ip,string operation) { string resJson = CreateJson(code, msg, 0); cout<<"response result:"<::iterator it_already_build) * @param {const} shared_ptr * @param {string} db_name */ -void load_thread_new(const shared_ptr& response, string db_name) +void load_thread_new(const shared_ptr& response, string db_name,string ip, bool load_csr) { + string operation="load"; string error = checkparamValue("db_name", db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } if (checkdbexist(db_name)==false) { error = "the database ["+db_name+"] not built yet."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } - if (checkdbload(db_name)) + pthread_rwlock_rdlock(&databases_map_lock); + std::map::iterator it = databases.find(db_name); + pthread_rwlock_unlock(&databases_map_lock); + if (databases.find(db_name) != databases.end()) { - error = "the database already load yet."; - sendResponseMsg(0, error, response); + Database * current_database = it->second; + string csr_str = "false"; + if (current_database->csr != NULL) + { + csr_str = "true"; + } + rapidjson::Document doc; + doc.SetObject(); + Document::AllocatorType &allocator = doc.GetAllocator(); + doc.AddMember("StatusCode", 0, allocator); + doc.AddMember("StatusMsg", "the database already load yet.", allocator); + doc.AddMember("csr", StringRef(csr_str.c_str()), allocator); + rapidjson::StringBuffer resBuffer; + rapidjson::Writer resWriter(resBuffer); + doc.Accept(resWriter); + string json_str = resBuffer.GetString(); + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << json_str.length() << "\r\n\r\n" << json_str; return; } + pthread_rwlock_rdlock(&already_build_map_lock); std::map::iterator it_already_build = already_build.find(db_name); @@ -1551,7 +1639,7 @@ void load_thread_new(const shared_ptr& response, string db if (trylockdb(it_already_build)==false) { error = "the operation can not been excuted due to loss of lock."; - sendResponseMsg(1007, error, response); + sendResponseMsg(1007, error, response,ip,operation); pthread_rwlock_unlock(&already_build_map_lock); return; } @@ -1562,14 +1650,14 @@ void load_thread_new(const shared_ptr& response, string db { error = "Unable to load due to loss of lock"; pthread_rwlock_unlock(&it_already_build->second->db_lock); - sendResponseMsg(1007,error,response); + sendResponseMsg(1007,error,response,ip,operation); return; } cout<<"begin loading..."<load(); + bool flag = current_database->load(load_csr); cout<<"end loading..."< txn_m = make_shared(current_database, database); pthread_rwlock_wrlock(&txn_m_lock); @@ -1583,7 +1671,7 @@ void load_thread_new(const shared_ptr& response, string db if (!flag) { error = "Failed to load the database."; - sendResponseMsg(1005, error, response); + sendResponseMsg(1005, error, response,ip,operation); return; } // if(pthread_rwlock_trywrlock(&databases_map_lock)!=0) @@ -1601,11 +1689,25 @@ void load_thread_new(const shared_ptr& response, string db databases.insert(pair(db_name, current_database)); cout<<"insert the pair into the databases"<second->db_lock)); + + cout << "database insert done." << endl; + string csr_str = "false"; + if (current_database->csr != NULL) + { + csr_str = "true"; + } + rapidjson::Document doc; + doc.SetObject(); + Document::AllocatorType &allocator = doc.GetAllocator(); + doc.AddMember("StatusCode", 0, allocator); + doc.AddMember("StatusMsg", "database loaded successfully.", allocator); + doc.AddMember("csr", StringRef(csr_str.c_str()), allocator); + rapidjson::StringBuffer resBuffer; + rapidjson::Writer resWriter(resBuffer); + doc.Accept(resWriter); + string json_str = resBuffer.GetString(); + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << json_str.length() << "\r\n\r\n" << json_str; } @@ -1620,19 +1722,20 @@ void load_thread_new(const shared_ptr& response, string db * @param {string} db_name * @return {*} */ -void monitor_thread_new(const shared_ptr& response, string db_name) +void monitor_thread_new(const shared_ptr& response, string db_name,string ip) { //check the param value is legal or not. + string operation="monitor"; string error=checkparamValue("db_name",db_name); if(error.empty()==false) { - sendResponseMsg(1003,error,response); + sendResponseMsg(1003,error,response,ip,operation); return; } if (checkdbexist(db_name)==false) { error = "the database ["+db_name+"] not built yet."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } // if (checkdbload(db_name)) @@ -1647,7 +1750,7 @@ void monitor_thread_new(const shared_ptr& response, string { //cout << "database not loaded yet." << endl; error = "Database not load yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&databases_map_lock); return; } @@ -1708,6 +1811,7 @@ void monitor_thread_new(const shared_ptr& response, string //*response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success; pthread_rwlock_unlock(&(it_already_build->second->db_lock)); + writeIpAccessLog(ip,operation,"monitor "+db_name+" successfully",0); } /** @@ -1717,25 +1821,26 @@ void monitor_thread_new(const shared_ptr& response, string * @param {string} db_name the database name * @return {*} */ -void unload_thread_new(const shared_ptr &response, string db_name) +void unload_thread_new(const shared_ptr &response, string db_name,string ip) { + string operation="unload"; string error = checkparamValue("db_name", db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } if (checkdbexist(db_name) == false) { error = "the database [" + db_name + "] not built yet."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } if (checkdbload(db_name) == false) { error = "the database not load yet."; - sendResponseMsg(0, error, response); + sendResponseMsg(0, error, response,ip,operation); return; } pthread_rwlock_rdlock(&already_build_map_lock); @@ -1744,7 +1849,7 @@ void unload_thread_new(const shared_ptr &response, string if (trylockdb(it_already_build) == false) { error = "the operation can not been excuted due to loss of lock."; - sendResponseMsg(1007, error, response); + sendResponseMsg(1007, error, response,ip,operation); return; } else @@ -1754,7 +1859,7 @@ void unload_thread_new(const shared_ptr &response, string if (iter == databases.end()) { string error = "Database not load yet."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); pthread_rwlock_unlock(&databases_map_lock); pthread_rwlock_unlock(&(it_already_build->second->db_lock)); return; @@ -1764,7 +1869,7 @@ void unload_thread_new(const shared_ptr &response, string if (txn_managers.find(db_name) == txn_managers.end()) { error = "transaction manager can not find the database"; - sendResponseMsg(1008, error, response); + sendResponseMsg(1008, error, response,ip,operation); pthread_rwlock_unlock(&txn_m_lock); pthread_rwlock_unlock(&(it_already_build->second->db_lock)); //*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; @@ -1777,8 +1882,8 @@ void unload_thread_new(const shared_ptr &response, string current_database = NULL; databases.erase(db_name); - string success = "Database unloaded."; - sendResponseMsg(0, success, response); + string success = "Database ["+db_name+"] unloaded."; + sendResponseMsg(0, success, response,ip,operation); //*response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success; // pthread_rwlock_unlock(&database_load_lock); pthread_rwlock_unlock(&databases_map_lock); @@ -1795,18 +1900,19 @@ void unload_thread_new(const shared_ptr &response, string * @param {string} is_backup: true:logical drop,the file is not deleted; false:force drop, the file also is deleted. * @return {*} */ -void drop_thread_new(const shared_ptr &response, string db_name, string is_backup) +void drop_thread_new(const shared_ptr &response, string db_name, string is_backup,string ip) { + string operation="drop"; string error = checkparamValue("db_name", db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } if (checkdbexist(db_name) == false) { error = "the database [" + db_name + "] not built yet."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } pthread_rwlock_rdlock(&already_build_map_lock); @@ -1815,12 +1921,12 @@ void drop_thread_new(const shared_ptr &response, string db if(it_already_build == already_build.end()) { error = "the database [" + db_name + "] not built yet."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); } if (trylockdb(it_already_build) == false) { error = "the operation can not been excuted due to loss of lock."; - sendResponseMsg(1007, error, response); + sendResponseMsg(1007, error, response,ip,operation); } else { @@ -1873,7 +1979,7 @@ void drop_thread_new(const shared_ptr &response, string db Util::delete_backuplog(db_name); string success = "Database " + db_name + " dropped."; - sendResponseMsg(0, success, response); + sendResponseMsg(0, success, response,ip,operation); return; } } @@ -1883,8 +1989,9 @@ void drop_thread_new(const shared_ptr &response, string db * @param {const shared_ptr} &response * @return {*} */ -void show_thread_new(const shared_ptr &response) +void show_thread_new(const shared_ptr &response,string username,string ip) { + string operation="show"; pthread_rwlock_rdlock(&already_build_map_lock); std::map::iterator it_already_build; string success; @@ -1903,6 +2010,11 @@ void show_thread_new(const shared_ptr &response) string time = it_already_build->second->getTime(); if ((database_name == "system")) continue; + if (username != ROOT_USERNAME && creator != username && checkPrivilege(username, "query", database_name) == false) + { + continue; + } + Value obj(kObjectType); Value _database_name; @@ -1930,12 +2042,13 @@ void show_thread_new(const shared_ptr &response) resDoc.Accept(resWriter); string resJson = resBuffer.GetString(); - *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json" << "\r\n\r\n" << resJson; - + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; + //*response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success; //return true; pthread_rwlock_unlock(&already_build_map_lock); + writeIpAccessLog(ip,operation,"show successfully!",0); return; } @@ -1948,19 +2061,22 @@ void show_thread_new(const shared_ptr &response) * @param {string} type :the operation type, 1: addUser 2:deleteUser 3:alterUser * @return {*} */ -void userManager_thread_new(const shared_ptr& response,string username,string password,string type) +void userManager_thread_new(const shared_ptr& response,string username,string password,string type,string ip) { string error=""; - if(type=="1") - { - //add user - if(username.empty()||password.empty()) - { - error="the user name and password can not be empty while adding user."; - sendResponseMsg(1003,error,response); + int code = 0; + string operation="usermanager"; + if(type=="1") + { + //add user + if(username.empty()||password.empty()) + { + error="the user name and password can not be empty while adding user."; + sendResponseMsg(1003,error,response,ip,operation); return; - } - else{ + } + else + { pthread_rwlock_wrlock(&users_map_lock); if(users.find(username) == users.end()) { @@ -1970,89 +2086,82 @@ void userManager_thread_new(const shared_ptr& response,str string update = "INSERT DATA {<" + username + "> \"" + password + "\".}"; updateSys(update); error="user add done."; - sendResponseMsg(0,error,response); + code = 0; } else { error = "username already existed, add user failed."; - sendResponseMsg(1004,error,response); - + code = 1004; } pthread_rwlock_unlock(&users_map_lock); + sendResponseMsg(code,error,response,ip,operation); return; - } - } - else if(type=="2") - { - //delete user + } + } + else if(type=="2") + { + //delete user pthread_rwlock_wrlock(&users_map_lock); std::map::iterator iter; iter = users.find(username); if(iter != users.end() && username != ROOT_USERNAME) { - delete iter->second; - iter->second = NULL; - users.erase(username); - string update = "DELETE where {<" + username + "> ?p ?o.}"; - updateSys(update); - sendResponseMsg(0,"delete user done.",response); - - } - else - { - string error; - int error_code; - if(username == ROOT_USERNAME) - { - error = "you cannot delete root, delete user failed."; - - } - else - { - error = "username not exist, delete user failed."; - - } - sendResponseMsg(1004,error,response); - pthread_rwlock_unlock(&users_map_lock); - return; - } - pthread_rwlock_unlock(&users_map_lock); - return; - - } - else if(type=="3") - { - pthread_rwlock_wrlock(&users_map_lock); - std::map::iterator iter; - iter = users.find(username); - - if(iter == users.end()) + delete iter->second; + iter->second = NULL; + users.erase(username); + string update = "DELETE where {<" + username + "> ?p ?o.}"; + updateSys(update); + error = "delete user done."; + code = 0; + } + else + { + if(username == ROOT_USERNAME) { - error = "username not exist, change password failed."; - sendResponseMsg(1004,error,response); - pthread_rwlock_unlock(&users_map_lock); - return; - + error = "you cannot delete root, delete user failed."; } else { - iter->second->setPassword(password); - string update = "DELETE WHERE {<" + username + "> ?o.}"; - updateSys(update); - string update2 = "INSERT DATA {<" + username + "> \"" + password + "\".}"; - updateSys(update2); - sendResponseMsg(0,"change password done.",response); - + error = "username not exist, delete user failed."; } - pthread_rwlock_unlock(&users_map_lock); - return; - } - else - { - error="the operation is not support."; - sendResponseMsg(1003,error,response); - return; - } + code = 1004; + } + pthread_rwlock_unlock(&users_map_lock); + sendResponseMsg(code,error,response,ip,operation); + return; + } + else if(type=="3") + { + pthread_rwlock_wrlock(&users_map_lock); + std::map::iterator iter; + iter = users.find(username); + + if(iter == users.end()) + { + error = "username not exist, change password failed."; + code = 1004; + } + else + { + iter->second->setPassword(password); + string update = "DELETE WHERE {<" + username + "> ?o.}"; + updateSys(update); + string update2 = "INSERT DATA {<" + username + "> \"" + password + "\".}"; + updateSys(update2); + error = "change password done."; + code = 0; + } + pthread_rwlock_unlock(&users_map_lock); + sendResponseMsg(code,error,response,ip,operation); + return; + } + else + { + error = "the operation is not support."; + code = 1003; + sendResponseMsg(code,error,response,ip,operation); + return; + } } /** @@ -2061,13 +2170,14 @@ void userManager_thread_new(const shared_ptr& response,str * @param {const} shared_ptr * @return {*} */ -void showuser_thread_new(const shared_ptr& response) +void showuser_thread_new(const shared_ptr& response,string ip) { + string operation="showuser"; pthread_rwlock_rdlock(&users_map_lock); if (users.empty()) { string error = "No Users."; - sendResponseMsg(0,error,response); + sendResponseMsg(0,error,response,ip,operation); //*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; pthread_rwlock_unlock(&users_map_lock); return; @@ -2141,6 +2251,7 @@ void showuser_thread_new(const shared_ptr& response) //*response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success; pthread_rwlock_unlock(&users_map_lock); + writeIpAccessLog(ip,operation,"show user successfully!",0); return; } @@ -2155,19 +2266,20 @@ void showuser_thread_new(const shared_ptr& response) * @return {*} */ void userPrivilegeManage_thread_new(const shared_ptr &response, string username, - string privilege, string type, string db_name) + string privilege, string type, string db_name,string ip) { string error = ""; + string operation="userPrivilegeManage"; error=checkparamValue("type",type); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } error=checkparamValue("username",username); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } @@ -2176,13 +2288,13 @@ void userPrivilegeManage_thread_new(const shared_ptr &resp error = checkparamValue("db_name", db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } error = checkparamValue("privilege", privilege); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } } @@ -2190,7 +2302,7 @@ void userPrivilegeManage_thread_new(const shared_ptr &resp if (username == ROOT_USERNAME) { string error = "you can't add privilege to root user."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); //*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; return; } @@ -2202,7 +2314,7 @@ void userPrivilegeManage_thread_new(const shared_ptr &resp if(resultint==-1) { error="the username is not exists or the username is root."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } else @@ -2280,13 +2392,13 @@ void userPrivilegeManage_thread_new(const shared_ptr &resp else { result = "the operation type is not support."; - sendResponseMsg(1003, result, response); + sendResponseMsg(1003, result, response,ip,operation); return; } } } } - sendResponseMsg(0, result, response); + sendResponseMsg(0, result, response,ip,operation); return; } @@ -2298,19 +2410,20 @@ void userPrivilegeManage_thread_new(const shared_ptr &resp * @param {string} backup_path: the backup path * @return {*} */ -void backup_thread_new(const shared_ptr& response,string db_name,string backup_path) +void backup_thread_new(const shared_ptr& response,string db_name,string backup_path,string ip) { string error=""; + string operation="backup"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } if (checkdbexist(db_name) == false) { error = "the database [" + db_name + "] not built yet."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } pthread_rwlock_rdlock(&already_build_map_lock); @@ -2319,7 +2432,7 @@ void backup_thread_new(const shared_ptr& response,string d if (trylockdb(it_already_build) == false) { error = "the operation can not been excuted due to loss of lock."; - sendResponseMsg(1007, error, response); + sendResponseMsg(1007, error, response,ip,operation); return; } //begin backup database @@ -2328,11 +2441,11 @@ void backup_thread_new(const shared_ptr& response,string d if(path == "." ){ cout << "Backup Path Can not be root or empty, Backup Failed!" << endl; string error = "Failed to backup the database. Backup Path Can not be root or empty."; - sendResponseMsg(1003,error,response); + sendResponseMsg(1003,error,response,ip,operation); pthread_rwlock_unlock(&(it_already_build->second->db_lock)); return; } - if(path[0] == '.') path = path.substr(1, path.length() - 1); + //if(path[0] == '.') path = path.substr(1, path.length() - 1); //if(path[0] == '/') path = path.substr(1, path.length() - 1); if(path[path.length() - 1] == '/') path = path.substr(0, path.length() - 1); cout<<"backup path:"<& response,string d if(ret == 1){ cout << "Database Folder Misssing, Backup Failed!" << endl; string error = "Failed to backup the database. Database Folder Misssing."; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); pthread_rwlock_unlock(&(it_already_build->second->db_lock)); return; } @@ -2377,6 +2490,7 @@ void backup_thread_new(const shared_ptr& response,string d *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; pthread_rwlock_unlock(&(it_already_build->second->db_lock)); + writeIpAccessLog(ip,operation,"backup "+db_name+" database successfully!",0); } @@ -2389,13 +2503,14 @@ void backup_thread_new(const shared_ptr& response,string d * @param {string} username the operation username * @return {*} */ -void restore_thread_new(const shared_ptr& response,string db_name,string backup_path,string username) +void restore_thread_new(const shared_ptr& response,string db_name,string backup_path,string username,string ip) { string error=""; + string operation="restore"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } string path=backup_path; @@ -2404,7 +2519,7 @@ void restore_thread_new(const shared_ptr& response,string cout<<"backup path:"<& response,string string time = Util::get_backup_time(path, db_name); if(time.size() == 0){ string error = "Backup Path Does not Match DataBase Name, Restore Failed"; - sendResponseMsg(1003,error,response); + sendResponseMsg(1003,error,response,ip,operation); pthread_rwlock_unlock(&already_build_map_lock); return; } @@ -2431,7 +2546,7 @@ void restore_thread_new(const shared_ptr& response,string { string error = "add query or load or unload or backup or restore privilege failed."; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); //*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; //return false; @@ -2454,7 +2569,7 @@ void restore_thread_new(const shared_ptr& response,string { string error = "Unable to restore due to loss of lock"; - sendResponseMsg(1007,error,response); + sendResponseMsg(1007,error,response,ip,operation); //*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; return; @@ -2470,7 +2585,7 @@ void restore_thread_new(const shared_ptr& response,string if(ret == 1){ string error = "Failed to restore the database. Backup Path Error"; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); cout << "Backup Path Error, Restore Failed!" << endl; pthread_rwlock_unlock(&(it_already_build->second->db_lock)); return; @@ -2487,8 +2602,8 @@ void restore_thread_new(const shared_ptr& response,string system(sys_cmd.c_str()); cout << "database restore done." << endl; - string success = "Database restore successfully."; - sendResponseMsg(0,success,response); + string success = "Database ["+db_name+"] restore successfully."; + sendResponseMsg(0,success,response,ip,operation); } /** @@ -2506,28 +2621,29 @@ string db_name,string sparql,string format, string update_flag,string remote_ip,string log_prefix) { string error=""; + string operation="query"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,remote_ip,operation); return; } error=checkparamValue("sparql",sparql); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,remote_ip,operation); return; } if(checkdbexist(db_name)==false) { error="Database not build yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,remote_ip,operation); return; } if(checkdbload(db_name)==false) { error="Database not load yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,remote_ip,operation); return; } bool update_flag_bool=true; @@ -2545,7 +2661,7 @@ string update_flag,string remote_ip,string log_prefix) { string error = "Database not load yet."; //cout << error << endl; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,remote_ip,operation); pthread_rwlock_unlock(&databases_map_lock); return; } @@ -2616,7 +2732,7 @@ string update_flag,string remote_ip,string log_prefix) string content=exception_msg; cout << exception_msg; - sendResponseMsg(1005,content,response); + sendResponseMsg(1005,content,response,remote_ip,operation); pthread_rwlock_unlock(&(it_already_build->second->db_lock)); return; } @@ -2624,7 +2740,7 @@ string update_flag,string remote_ip,string log_prefix) { string content =e2.what(); cout<<"query failed:"<second->db_lock)); return; } @@ -2632,7 +2748,7 @@ string update_flag,string remote_ip,string log_prefix) { string content = "unknow error"; cout<<"query failed:"<second->db_lock)); return; @@ -2695,9 +2811,9 @@ string update_flag,string remote_ip,string log_prefix) // string QueryTime = Util::int2string(query_time) + "ms"; doc.AddMember("QueryTime", query_time, doc_allocator); StringBuffer buffer; - PrettyWriter writer(buffer); + Writer writer(buffer); doc.Accept(writer); - writeLog(query_logfp, buffer.GetString()); + writeLog(buffer.GetString()); } //string log_info = Util::get_date_time() + "\n" + sparql + "\n\nanswer num: " + Util::int2string(rs.ansNum)+"\nquery time: "+Util::int2string(query_time) +" ms\n-----------------------------------------------------------\n"; @@ -2715,6 +2831,7 @@ string update_flag,string remote_ip,string log_prefix) string success = ""; //TODO: if result is stored in Stream instead of memory? (if out of memory to use to_str) //BETTER: divide and transfer, in multiple times, getNext() + writeIpAccessLog(remote_ip,operation,"query successfully!",0); if (format == "json") { success = rs.to_JSON(); @@ -2737,7 +2854,7 @@ string update_flag,string remote_ip,string log_prefix) outfile.open(localname2); outfile << success; outfile.close(); - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,remote_ip,operation); return; } @@ -2844,7 +2961,7 @@ string update_flag,string remote_ip,string log_prefix) { cout<<"result parse error:"<second->db_lock)); //return false; @@ -2943,25 +3060,26 @@ string update_flag,string remote_ip,string log_prefix) * @param {string} username * @return {*} */ -void export_thread_new(const shared_ptr& response,string db_name,string db_path,string username) +void export_thread_new(const shared_ptr& response,string db_name,string db_path,string username,string ip) { string error=""; + string operation="export"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } error=checkparamValue("db_path",db_path); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } if(checkdbexist(db_name)==false) { error="Database not build yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); return; } if(db_path[db_path.length()-1] != '/') @@ -2977,7 +3095,7 @@ void export_thread_new(const shared_ptr& response,string d if(iter == databases.end()) { string error = "Database not load yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&databases_map_lock); return; } @@ -2993,7 +3111,7 @@ void export_thread_new(const shared_ptr& response,string d ResultSet rs; cout<<"db_path:"<query(sparql, rs, ofp, true, true); + int ret = current_database->query(sparql, rs, ofp, false, false); fflush(ofp); fclose(ofp); ofp = NULL; @@ -3016,6 +3134,8 @@ void export_thread_new(const shared_ptr& response,string d pthread_rwlock_unlock(&(it_already_build->second->db_lock)); + writeIpAccessLog(ip,operation,"Database "+db_name+" export successfully!",0); + // string resJson = CreateJson(0, success, 0); // *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; // sendResponseMsg(0,success,response); @@ -3031,9 +3151,10 @@ void export_thread_new(const shared_ptr& response,string d * @param {*} * @return {*} */ -void login_thread_new(const shared_ptr& response) +void login_thread_new(const shared_ptr& response,string ip) { string success="login successfully."; + string operation="login"; Document resDoc; resDoc.SetObject(); Document::AllocatorType &allocator = resDoc.GetAllocator(); @@ -3048,6 +3169,7 @@ void login_thread_new(const shared_ptr& response) PrettyWriter resWriter(resBuffer); resDoc.Accept(resWriter); string resJson = resBuffer.GetString(); + writeIpAccessLog(ip,operation,success,0); *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; @@ -3060,10 +3182,10 @@ void login_thread_new(const shared_ptr& response) * @param {*} * @return {*} */ -void check_thread_new(const shared_ptr& response) +void check_thread_new(const shared_ptr& response,string ip) { string success="the ghttp server is running..."; - sendResponseMsg(0,success,response); + sendResponseMsg(0,success,response,ip,"check"); } /** * @Author: liwenjie @@ -3074,32 +3196,33 @@ void check_thread_new(const shared_ptr& response) * @param {string} isolevel : the Isolation level， 1:RC(read committed) 2:SI(snapshot isolation) 3:SR(seriablizable） * @return {*} */ -void begin_thread_new(const shared_ptr& response,string db_name,string isolevel,string username) +void begin_thread_new(const shared_ptr& response,string db_name,string isolevel,string username,string ip) { string error=""; + string operation="begin"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } error=checkparamValue("isolevel",isolevel); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } int level=Util::string2int(isolevel); if(level<=0||level>3) { error="the Isolation level's value only can been 1/2/3"; - sendResponseMsg(1003,error,response); + sendResponseMsg(1003,error,response,ip,operation); return; } if (checkdbexist(db_name) == false) { error = "the database [" + db_name + "] not built yet."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } @@ -3108,7 +3231,7 @@ void begin_thread_new(const shared_ptr& response,string db if (iter == databases.end()) { error = "Database not load yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&databases_map_lock); return; } @@ -3118,7 +3241,7 @@ void begin_thread_new(const shared_ptr& response,string db if (txn_managers.find(db_name) == txn_managers.end()) { error = "Database transaction manager error."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&txn_m_lock); return; } @@ -3131,12 +3254,12 @@ void begin_thread_new(const shared_ptr& response,string db cout <<"Transcation Id:"<< to_string(TID) << endl; cout << to_string(txn_m->Get_Transaction(TID)->GetStartTime()) << endl; string begin_time = to_string(txn_m->Get_Transaction(TID)->GetStartTime()); - string Time_TID = begin_time + " " + to_string(TID); + string Time_TID = begin_time + "_" + to_string(TID); Util::add_transactionlog(db_name, username, Time_TID, begin_time, "RUNNING", "INF"); if (TID == INVALID_ID) { error = "transaction begin failed."; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); return; } //TODO: write the transaction log @@ -3157,6 +3280,7 @@ void begin_thread_new(const shared_ptr& response,string db writer.EndObject(); string resJson = s.GetString(); //string resJson = CreateJson(992, success, 0); + writeIpAccessLog(ip,operation,"transaction begin success,tid:"+TID,0); *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; } @@ -3170,24 +3294,29 @@ void begin_thread_new(const shared_ptr& response,string db * @param {string} sparql:the query sparql (only support insert and delete) * @return {*} */ -void tquery_thread_new(const shared_ptr& response,string db_name,string TID_s,string sparql) +void tquery_thread_new(const shared_ptr& response,string db_name,string TID_s,string sparql,string ip) { string error=""; + string operation="tquery"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } error=checkparamValue("TID",TID_s); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } txn_id_t TID; - + if (TID_s.find("_") != string::npos) + { + int pos = TID_s.find("_") + 1; + TID_s = TID_s.substr(pos, TID_s.size()-pos); + } if(Util::is_number(TID_s)) { TID = strtoull(TID_s.c_str(), NULL, 0); @@ -3195,13 +3324,13 @@ void tquery_thread_new(const shared_ptr& response,string d else { error = "TID is not a pure number. TID: " + TID_s; - sendResponseMsg(1003,error,response); + sendResponseMsg(1003,error,response,ip,operation); return; } error=checkparamValue("sparql",sparql); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } cout << "try to get databases_map_lock" << endl; @@ -3210,7 +3339,7 @@ void tquery_thread_new(const shared_ptr& response,string d if (iter == databases.end()) { error = "Database not load yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&databases_map_lock); return; } @@ -3219,7 +3348,7 @@ void tquery_thread_new(const shared_ptr& response,string d if (txn_managers.find(db_name) == txn_managers.end()) { error = "Database transaction manager error."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&txn_m_lock); return; } @@ -3231,19 +3360,19 @@ void tquery_thread_new(const shared_ptr& response,string d if(ret == -1) { error = "Transaction query failed due to wrong TID"; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); return; } else if(ret == -10) { error = "Transaction query failed due to wrong database status"; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); return; } else if(ret == -99) { error = "Transaction query failed. This transaction is not in running status!"; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); return; } else if(ret == -100) @@ -3279,24 +3408,25 @@ void tquery_thread_new(const shared_ptr& response,string d *response << "HTTP/1.1 200 OK\r\nContent-Length: " << res.length(); *response << "\r\nContent-Type: application/octet-stream"; *response << "\r\n\r\n" << res; + writeIpAccessLog(ip,operation,"tquery successfully!",0); return; } else if(ret == -20) { error = "Transaction query failed. This transaction is set abort due to conflict!"; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); return; } else if(ret == -101) { error = "Transaction query failed. Unknown query error"; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); return; } else { string success = "Transaction query success, update num: " + Util::int2string(ret); - sendResponseMsg(0,success,response); + sendResponseMsg(0,success,response,ip,operation); return; } } @@ -3309,45 +3439,50 @@ void tquery_thread_new(const shared_ptr& response,string d * @param {string} TID_s * @return {*} */ -void commit_thread_new(const shared_ptr& response,string db_name,string TID_s) +void commit_thread_new(const shared_ptr& response,string db_name,string TID_s,string ip) { string error=""; + string operation="commit"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } error=checkparamValue("TID",TID_s); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } string res; txn_id_t TID; - + if (TID_s.find("_") != string::npos) + { + int pos = TID_s.find("_") + 1; + TID_s = TID_s.substr(pos, TID_s.size()-pos); + } if(Util::is_number(TID_s)) { TID = strtoull(TID_s.c_str(), NULL, 0); } else { - error = "TID is not a pure number. TID: " + TID_s; - sendResponseMsg(1003,error,response); + error = "TID is not a pure number. TID: " + TID_s; + sendResponseMsg(1003,error,response,ip,operation); return; } if(checkdbexist(db_name)==false) { error = "Database not built yet. "; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); return; } if(checkdbload(db_name)==false) { error="Database not load yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); return; } @@ -3356,7 +3491,7 @@ void commit_thread_new(const shared_ptr& response,string d if (txn_managers.find(db_name) == txn_managers.end()) { string error = "Database transaction manager error."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&txn_m_lock); return; } @@ -3370,22 +3505,22 @@ void commit_thread_new(const shared_ptr& response,string d if (ret == 1) { error = "transaction not in running state! commit failed. TID: " + TID_s; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); return; } else if (ret == -1) { error = "transaction not found, commit failed. TID: " + TID_s; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); return; } else { string begin_time = to_string(txn_m->Get_Transaction(TID)->GetStartTime()); - string Time_TID = begin_time + " " + to_string(TID); + string Time_TID = begin_time + "_" + to_string(TID); Util::update_transactionlog(Time_TID, "COMMITED", to_string(txn_m->Get_Transaction(TID)->GetEndTime())); string success = "transaction commit success. TID: " + TID_s; - sendResponseMsg(0,success,response); + sendResponseMsg(0,success,response,ip,operation); return; } @@ -3400,54 +3535,58 @@ void commit_thread_new(const shared_ptr& response,string d * @param {string} TID_s * @return {*} */ -void rollback_thread_new(const shared_ptr& response,string db_name,string TID_s) +void rollback_thread_new(const shared_ptr& response,string db_name,string TID_s,string ip) { string error=""; + string operation="rollback"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } error=checkparamValue("TID",TID_s); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } string res; txn_id_t TID; - + if (TID_s.find("_") != string::npos) + { + int pos = TID_s.find("_") + 1; + TID_s = TID_s.substr(pos, TID_s.size()-pos); + } if(Util::is_number(TID_s)) { TID = strtoull(TID_s.c_str(), NULL, 0); } else { - error = "TID is not a pure number. TID: " + TID_s; - sendResponseMsg(1003,error,response); + error = "TID is not a pure number. TID: " + TID_s; + sendResponseMsg(1003,error,response,ip,operation); return; } if(checkdbexist(db_name)==false) { error = "Database not built yet. "; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); return; } if(checkdbload(db_name)==false) { error="Database not load yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); return; } - pthread_rwlock_rdlock(&txn_m_lock); if (txn_managers.find(db_name) == txn_managers.end()) { string error = "Database transaction manager error."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&txn_m_lock); return; } @@ -3461,22 +3600,22 @@ void rollback_thread_new(const shared_ptr& response,string if (ret == 1) { error = "transaction not in running state! rollback failed. TID: " + TID_s; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); return; } else if (ret == -1) { error = "transaction not found, rollback failed. TID: " + TID_s; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,ip,operation); return; } else { string begin_time = to_string(txn_m->Get_Transaction(TID)->GetStartTime()); - string Time_TID = begin_time + " " + to_string(TID); + string Time_TID = begin_time + "_" + to_string(TID); Util::update_transactionlog(Time_TID, "ROLLBACK", to_string(txn_m->Get_Transaction(TID)->GetEndTime())); string success = "transaction rollback success. TID: " + TID_s; - sendResponseMsg(0,success,response); + sendResponseMsg(0,success,response,ip,operation); return; } } @@ -3490,17 +3629,19 @@ void rollback_thread_new(const shared_ptr& response,string * @param {*} * @return {*} */ -void txnlog_thread_new(const shared_ptr& response,string username) +void txnlog_thread_new(const shared_ptr& response,string username, int page_no, int page_size,string ip) { + string operation="txnlog"; if(username==ROOT_USERNAME) { - string resJson = Util::get_transactionlog(); + string resJson = Util::get_transactionlog(page_no, page_size); + writeIpAccessLog(ip,operation,"get txnlog successfully!",0); *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; } else { string error = "Root User Only!"; - sendResponseMsg(1003,error,response); + sendResponseMsg(1003,error,response,ip,operation); } } @@ -3512,19 +3653,20 @@ void txnlog_thread_new(const shared_ptr& response,string u * @param {string} db_name * @return {*} */ -void checkpoint_thread_new(const shared_ptr& response,string db_name) +void checkpoint_thread_new(const shared_ptr& response,string db_name,string ip) { string error; + string operation="checkpoint"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } if(checkdbexist(db_name)==false) { error="database not built yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); return; } @@ -3533,7 +3675,7 @@ void checkpoint_thread_new(const shared_ptr& response,stri if(iter == databases.end()) { error = "Database not load yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&databases_map_lock); return; } @@ -3546,7 +3688,7 @@ void checkpoint_thread_new(const shared_ptr& response,stri if (trylockdb(it_already_build)==false) { error = "the operation can not been excuted due to loss of lock."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } else @@ -3555,7 +3697,7 @@ void checkpoint_thread_new(const shared_ptr& response,stri if (txn_managers.find(db_name) == txn_managers.end()) { error = "Database transaction manager error."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&txn_m_lock); return; } @@ -3567,7 +3709,7 @@ void checkpoint_thread_new(const shared_ptr& response,stri //NOTICE: this info is in header string success = "Database saved successfully."; //header and content are split by an empty line - sendResponseMsg(0,success,response); + sendResponseMsg(0,success,response,ip,operation); pthread_rwlock_unlock(&(it_already_build->second->db_lock)); return; @@ -3576,9 +3718,10 @@ void checkpoint_thread_new(const shared_ptr& response,stri } - void test_connect_thread_new(const shared_ptr& response) + void test_connect_thread_new(const shared_ptr& response,string ip) { Document resDoc; + string operation="testConnect"; resDoc.SetObject(); Document::AllocatorType &allocator = resDoc.GetAllocator(); resDoc.AddMember("StatusCode", 0, allocator); @@ -3592,14 +3735,15 @@ void checkpoint_thread_new(const shared_ptr& response,stri PrettyWriter resWriter(resBuffer); resDoc.Accept(resWriter); string resJson = resBuffer.GetString(); - + writeIpAccessLog(ip,operation,"successfully!",0); *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; } - void getCoreVersion_thread_new(const shared_ptr& response) + void getCoreVersion_thread_new(const shared_ptr& response,string ip) { + string operation="getCoreVersion"; Document resDoc; resDoc.SetObject(); Document::AllocatorType &allocator = resDoc.GetAllocator(); @@ -3612,7 +3756,7 @@ void checkpoint_thread_new(const shared_ptr& response,stri PrettyWriter resWriter(resBuffer); resDoc.Accept(resWriter); string resJson = resBuffer.GetString(); - + writeIpAccessLog(ip,operation,"Successfully!",0); *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; } @@ -3626,31 +3770,32 @@ void checkpoint_thread_new(const shared_ptr& response,stri * @param {string} file: the insert data file * @return {*} */ -void batchInsert_thread_new(const shared_ptr& response,string db_name,string file) +void batchInsert_thread_new(const shared_ptr& response,string db_name,string file,string ip) { string error; + string operation="batchInsert"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } error=checkparamValue("file",file); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } if(Util::file_exist(file)==false) { error="the data file is not exist"; - sendResponseMsg(1003,error,response); + sendResponseMsg(1003,error,response,ip,operation); return; } if(checkdbexist(db_name)==false) { error="database not built yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); return; } @@ -3659,7 +3804,7 @@ void batchInsert_thread_new(const shared_ptr& response,str if(iter == databases.end()) { error = "Database not load yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&databases_map_lock); return; } @@ -3672,7 +3817,7 @@ void batchInsert_thread_new(const shared_ptr& response,str if (trylockdb(it_already_build)==false) { error = "the operation can not been excuted due to loss of lock."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } else @@ -3693,6 +3838,7 @@ void batchInsert_thread_new(const shared_ptr& response,str //header and content are split by an empty line *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; pthread_rwlock_unlock(&(it_already_build->second->db_lock)); + writeIpAccessLog(ip,operation,"Batch Insert Successfully!",0); return; } } @@ -3706,31 +3852,32 @@ void batchInsert_thread_new(const shared_ptr& response,str * @param {string} file: the remove data file * @return {*} */ -void batchRemove_thread_new(const shared_ptr& response,string db_name,string file) +void batchRemove_thread_new(const shared_ptr& response,string db_name,string file,string ip) { string error; + string operation="batchRemove"; error=checkparamValue("db_name",db_name); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } error=checkparamValue("file",file); if (error.empty() == false) { - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,ip,operation); return; } if(Util::file_exist(file)==false) { error="the data file is not exist"; - sendResponseMsg(1003,error,response); + sendResponseMsg(1003,error,response,ip,operation); return; } if(checkdbexist(db_name)==false) { error="database not built yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); return; } @@ -3739,7 +3886,7 @@ void batchRemove_thread_new(const shared_ptr& response,str if(iter == databases.end()) { error = "Database not load yet."; - sendResponseMsg(1004,error,response); + sendResponseMsg(1004,error,response,ip,operation); pthread_rwlock_unlock(&databases_map_lock); return; } @@ -3752,7 +3899,7 @@ void batchRemove_thread_new(const shared_ptr& response,str if (trylockdb(it_already_build)==false) { error = "the operation can not been excuted due to loss of lock."; - sendResponseMsg(1004, error, response); + sendResponseMsg(1004, error, response,ip,operation); return; } else @@ -3777,12 +3924,13 @@ void batchRemove_thread_new(const shared_ptr& response,str //header and content are split by an empty line *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; pthread_rwlock_unlock(&(it_already_build->second->db_lock)); + writeIpAccessLog(ip,operation,"batchRemove successfully!",0); return; } } -string checkIdentity(string username, string password) +string checkIdentity(string username, string password,string is_encryption) { //check identity. pthread_rwlock_rdlock(&users_map_lock); @@ -3793,6 +3941,17 @@ string checkIdentity(string username, string password) pthread_rwlock_unlock(&users_map_lock); return error; } + if( is_encryption=="1") + { + if(Util::md5(it->second->getPassword())!=password) + { + string error = "wrong password."; + + pthread_rwlock_unlock(&users_map_lock); + return error; + } + + } else if (it->second->getPassword() != password) { string error = "wrong password."; @@ -3827,13 +3986,64 @@ string checkIdentity2(string username, string password,string port) } +bool checkIp(string ip) +{ + pthread_rwlock_rdlock(&ips_map_lock); + std::map::iterator it = ips.find(ip); + if(it != ips.end()) + { + pthread_rwlock_unlock(&ips_map_lock); + int errornum=it->second->getErrorNum(); + if(errornum>5) + { + return false; + } + + + } + else{ + + struct IpInfo *ipinfo=new IpInfo(ip); + ips.insert(pair(ip, ipinfo)); + pthread_rwlock_unlock(&ips_map_lock); + } + return true; +} + +void addErrorNumForIp(string ip) +{ +pthread_rwlock_rdlock(&ips_map_lock); + std::map::iterator it = ips.find(ip); + if(it != ips.end()) + { + + int errornum=it->second->getErrorNum(); + it->second->setErrorNum(errornum+1); + pthread_rwlock_unlock(&ips_map_lock); + } + else{ + + struct IpInfo *ipinfo=new IpInfo(ip); + ipinfo->setErrorNum(1); + ips.insert(pair(ip, ipinfo)); + pthread_rwlock_unlock(&ips_map_lock); + } + +} void request_thread(const shared_ptr& response, const shared_ptr& request, string RequestType) { + string remote_ip=getRemoteIp(request); if (!ipCheck(request)) { string error = "IP Blocked!"; - sendResponseMsg(1101, error, response); + sendResponseMsg(1101, error, response,remote_ip,"ipCheck"); + return; + } + if(!checkIp(remote_ip)) + { + string error = "The ip havs too many error during accessing ghttp, the ip has been locked until the ghttp restart!"; + sendResponseMsg(1101, error, response,remote_ip,"checkIp"); return; } string thread_id = Util::getThreadID(); @@ -3844,7 +4054,8 @@ const shared_ptr& request, string RequestType) string db_name; Document document; string url; - string remote_ip=getRemoteIp(request); + string is_encryption="0"; + //string remote_ip=getRemoteIp(request); cout<<"request time :"<method << endl; @@ -3866,6 +4077,7 @@ const shared_ptr& request, string RequestType) password = UrlDecode(password); db_name = WebUrl::CutParam(url, "db_name"); + is_encryption=WebUrl::CutParam(url,"encryption"); } else if (RequestType == "POST") @@ -3881,7 +4093,7 @@ const shared_ptr& request, string RequestType) if(document.HasParseError()) { string error = "the post content is not fit the json format,content=" + strJson; - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,remote_ip,operation); return; } @@ -3905,30 +4117,35 @@ const shared_ptr& request, string RequestType) { password=document["password"].GetString(); } + if(document.HasMember("encryption")) + { + is_encryption=document["encryption"].GetString(); + } } else { string msg = "The method type " + request->method + " is not support"; - sendResponseMsg(1000, msg, response); + sendResponseMsg(1000, msg, response,remote_ip,"Unkown Method"); return; } if(operation=="check") { - check_thread_new(response); + check_thread_new(response,remote_ip); return; } - string checkidentityresult = checkIdentity(username, password); + string checkidentityresult = checkIdentity(username, password,is_encryption); if (checkidentityresult.empty() == false) { - sendResponseMsg(1001, checkidentityresult, response); + addErrorNumForIp(remote_ip); + sendResponseMsg(1001, checkidentityresult, response,remote_ip,"CheckIdnetity"); return; } if (checkPrivilege(username, operation, db_name) == 0) { string msg = "You have no " + operation + " privilege, operation failed"; - sendResponseMsg(1002, msg, response); + sendResponseMsg(1002, msg, response,remote_ip,"CheckPrivilege"); return; } cout << log_prefix << "HTTP: this is " <& request, string RequestType) catch (...) { string error = "the parameter has some error,please look up the api document."; - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,remote_ip,"build"); return; } } - build_thread_new(response, db_name, db_path, username, password); + build_thread_new(response, db_name, db_path, username, password,remote_ip); } //load dababase else if (operation == "load") { - load_thread_new(response, db_name); + bool load_csr = false; + string loadCSRStr = ""; + try + { + if (RequestType == "GET") + { + loadCSRStr = WebUrl::CutParam(url, "load_csr"); + loadCSRStr = UrlDecode(loadCSRStr); + } + else if (RequestType == "POST") + { + if (document.HasMember("load_csr")&&document["load_csr"].IsString()) + { + loadCSRStr = document["load_csr"].GetString(); + } + } + } + catch (...) + { + string error = "the parameter has some error,please look up the api document."; + sendResponseMsg(1003, error, response, remote_ip, "load"); + return; + } + if (loadCSRStr == "true") + load_csr = true; + load_thread_new(response, db_name, remote_ip, load_csr); } //monitor database else if (operation == "monitor") { - monitor_thread_new(response,db_name); + monitor_thread_new(response,db_name,remote_ip); } //unload database else if(operation=="unload") { cout<<"unload txn_manager:"<& request, string RequestType) catch (...) { string error = "the parameter has some error,please look up the api document."; - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,remote_ip,"drop"); return; } } @@ -4009,12 +4251,12 @@ const shared_ptr& request, string RequestType) is_backup="true"; } cout<<"is_backup:"<& request, string RequestType) { string error = "the parameter has some error,please look up the api document."; - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,remote_ip,"usermanage"); return; } } - userManager_thread_new(response,op_username,op_password,type); + userManager_thread_new(response,op_username,op_password,type,remote_ip); } //show the user list else if(operation=="showuser") { - showuser_thread_new(response); + showuser_thread_new(response,remote_ip); } //manage user privilege else if(operation=="userprivilegemanage") @@ -4089,7 +4331,7 @@ const shared_ptr& request, string RequestType) privileges = document["privileges"].GetString(); } } - userPrivilegeManage_thread_new(response,op_username,privileges,type,db_name); + userPrivilegeManage_thread_new(response,op_username,privileges,type,db_name,remote_ip); } //backup a database else if(operation=="backup") @@ -4109,7 +4351,7 @@ const shared_ptr& request, string RequestType) } } - backup_thread_new(response,db_name,backup_path); + backup_thread_new(response,db_name,backup_path,remote_ip); } //restore database else if(operation=="restore") @@ -4127,7 +4369,7 @@ const shared_ptr& request, string RequestType) backup_path = document["backup_path"].GetString(); } } - restore_thread_new(response,db_name,backup_path,username); + restore_thread_new(response,db_name,backup_path,username,remote_ip); } //query database @@ -4200,11 +4442,11 @@ const shared_ptr& request, string RequestType) { db_path="."; } - export_thread_new(response,db_name,db_path,username); + export_thread_new(response,db_name,db_path,username,remote_ip); } else if(operation=="login") { - login_thread_new(response); + login_thread_new(response,remote_ip); } else if(operation=="begin") { @@ -4222,7 +4464,7 @@ const shared_ptr& request, string RequestType) isolevel = document["isolevel"].GetString(); } } - begin_thread_new(response,db_name,isolevel,username); + begin_thread_new(response,db_name,isolevel,username,remote_ip); } else if(operation=="tquery") { @@ -4247,7 +4489,7 @@ const shared_ptr& request, string RequestType) sparql = document["sparql"].GetString(); } } - tquery_thread_new(response,db_name,TID,sparql); + tquery_thread_new(response,db_name,TID,sparql,remote_ip); } else if(operation=="commit") @@ -4268,7 +4510,7 @@ const shared_ptr& request, string RequestType) } } - commit_thread_new(response,db_name,TID); + commit_thread_new(response,db_name,TID,remote_ip); } else if(operation=="rollback") { @@ -4288,24 +4530,53 @@ const shared_ptr& request, string RequestType) } } - rollback_thread_new(response,db_name,TID); + rollback_thread_new(response,db_name,TID,remote_ip); } else if(operation=="txnlog") { - txnlog_thread_new(response,username); + int page_no = 1; + int page_size = 10; + try + { + if (RequestType == "GET") + { + string str_page_no = WebUrl::CutParam(url, "pageNo"); + string str_page_size = WebUrl::CutParam(url, "pageSize"); + page_no = Util::string2int(str_page_no); + page_size = Util::string2int(str_page_size); + } + else if (RequestType == "POST") + { + if (document.HasMember("pageNo") && document["pageNo"].IsInt()) + { + page_no = document["pageNo"].GetInt(); + } + if (document.HasMember("pageSize") && document["pageSize"].IsInt()) + { + page_size = document["pageSize"].GetInt(); + } + } + } + catch (...) + { + string error = "the parameter has some error,please look up the api document."; + sendResponseMsg(1003, error, response, remote_ip, operation); + return; + } + txnlog_thread_new(response, username, page_no, page_size, remote_ip); } else if(operation=="checkpoint") { - checkpoint_thread_new(response,db_name); + checkpoint_thread_new(response,db_name,remote_ip); } else if(operation=="testConnect") { - test_connect_thread_new(response); + test_connect_thread_new(response,remote_ip); } else if(operation=="getCoreVersion") { - getCoreVersion_thread_new(response); + getCoreVersion_thread_new(response,remote_ip); } else if(operation=="batchInsert") @@ -4330,11 +4601,11 @@ const shared_ptr& request, string RequestType) catch (...) { string error = "the parameter has some error,please look up the api document."; - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,remote_ip,operation); return; } } - batchInsert_thread_new(response, db_name, file); + batchInsert_thread_new(response, db_name, file,remote_ip); } else if(operation=="batchRemove") { @@ -4358,11 +4629,11 @@ const shared_ptr& request, string RequestType) catch (...) { string error = "the parameter has some error,please look up the api document."; - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,remote_ip,"batchRemove"); return; } } - batchRemove_thread_new(response, db_name, file); + batchRemove_thread_new(response, db_name, file,remote_ip); } else if(operation=="querylog") { @@ -4386,48 +4657,224 @@ const shared_ptr& request, string RequestType) { file_name = document["date"].GetString(); } - if (document.HasMember("pageNo")&&document["pageNo"].IsInt()) + if (document.HasMember("pageNo")) { - page_no = document["pageNo"].GetInt(); + if (document["pageNo"].IsInt()) + { + page_no = document["pageNo"].GetInt(); + } + else if (document["pageNo"].IsString()) + { + page_no = Util::string2int(document["pageNo"].GetString()); + } } - if (document.HasMember("pageSize")&&document["pageSize"].IsInt()) + if (document.HasMember("pageSize")) { - page_size = document["pageSize"].GetInt(); + if (document["pageSize"].IsInt()) + { + page_size = document["pageSize"].GetInt(); + } + else if (document["pageSize"].IsString()) + { + page_size = Util::string2int(document["pageSize"].GetString()); + } } } - quereyLog_thread_new(response, file_name, page_no, page_size); + quereyLog_thread_new(response, file_name, page_no, page_size,remote_ip); } catch (...) { string error = "the parameter has some error,please look up the api document."; - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,remote_ip,"querylog"); return; } } - else { - string error="the operation "+operation +" has not match handler function"; - sendResponseMsg(1100, error, response); - } - - -} -/** - * @Author: liwenjie - * @param {const} HttpServer - * @param {const} shared_ptr - * @param {const} shared_ptr - * @param {string} RequestType - */ -void request_handler(const HttpServer& server, const shared_ptr& response, const shared_ptr& request, string RequestType) -{ - - // string operation; - // Document document; - // string url; - // string postcontent; - // if (RequestType == "GET") - // { - // cout << "request path:" << request->path << endl; + else if(operation=="accesslog") + { + string file_name = ""; + int page_no = 1; + int page_size = 10; + try + { + if (RequestType == "GET") + { + file_name = WebUrl::CutParam(url, "date"); + string str_page_no = WebUrl::CutParam(url, "pageNo"); + string str_page_size = WebUrl::CutParam(url, "pageSize"); + page_no = Util::string2int(str_page_no); + page_size = Util::string2int(str_page_size); + file_name = UrlDecode(file_name); + } + else if (RequestType == "POST") + { + if (document.HasMember("date")&&document["date"].IsString()) + { + file_name = document["date"].GetString(); + } + if (document.HasMember("pageNo")) + { + if (document["pageNo"].IsInt()) + { + page_no = document["pageNo"].GetInt(); + } + else if (document["pageNo"].IsString()) + { + page_no = Util::string2int(document["pageNo"].GetString()); + } + } + if (document.HasMember("pageSize")) + { + if (document["pageSize"].IsInt()) + { + page_size = document["pageSize"].GetInt(); + } + else if (document["pageSize"].IsString()) + { + page_size = Util::string2int(document["pageSize"].GetString()); + } + } + } + accessLog_thread_new(response, file_name, page_no, page_size,remote_ip); + } + catch (...) + { + string error = "the parameter has some error,please look up the api document."; + sendResponseMsg(1003, error, response,remote_ip,"accesslog"); + return; + } + } + else if(operation=="ipmanage") + { + string type = ""; + string ips = ""; + string ip_type = ""; + try + { + if (RequestType == "GET") + { + type = WebUrl::CutParam(url, "type"); + ips = WebUrl::CutParam(url, "ips"); + ip_type = WebUrl::CutParam(url, "ip_type"); + } + else if (RequestType == "POST") + { + if (document.HasMember("type")&&document["type"].IsString()) + { + type = document["type"].GetString(); + } + if (document.HasMember("ips")&&document["ips"].IsString()) + { + ips = document["ips"].GetString(); + } + if (document.HasMember("ip_type")&&document["ip_type"].IsString()) + { + ip_type = document["ip_type"].GetString(); + } + } + IPManager_thread_new(response, ips, ip_type, type); + } + catch (...) + { + string error = "the parameter has some error,please look up the api document."; + sendResponseMsg(1003, error, response,remote_ip,"ipmanage"); + return; + } + } + else { + string error="the operation "+operation +" has not match handler function"; + sendResponseMsg(1100, error, response,remote_ip,"UnkownOperation"); + } + + +} + +void writeIpAccessLog(string ip,string operation,string msg,int code) +{ + string iplog_name=Util::get_date_day(); + //cout << "querylog_name: " << querylog_name << endl; + //open the query log + if(Util::dir_exist(IP_ACCESS_PATH)==false) + { + Util::create_dir(IP_ACCESS_PATH); + } + string iplogfile = IP_ACCESS_PATH + iplog_name + ".log"; + if(Util::file_exist(iplogfile)==false) + { + cout<<"ip access log file is not exists, now create it."< writer(strBuf); + doc.Accept(writer); + string _info = strBuf.GetString(); + + _info.push_back(','); + _info.push_back('\n'); + + fprintf(ip_logfp, "%s", _info.c_str()); + + Util::Csync(ip_logfp); + long logSize = ftell(ip_logfp); + fclose(ip_logfp); + cout << "logSize: " << logSize << endl; + ip_log_lock.unlock(); +} + +void writeToBlackIPFile(string ip,string reason) +{ + if(ipBlackList) + { + ipBlackList->InsertIPToFile(ipBlackFile,ip,reason); + } +} + +void writeToWhiteIPFile(string ip,string reason) +{ + if(ipWhiteList) + { + ipWhiteList->InsertIPToFile(ipWhiteFile,ip,reason); + } +} +/** + * @Author: liwenjie + * @param {const} HttpServer + * @param {const} shared_ptr + * @param {const} shared_ptr + * @param {string} RequestType + */ +void request_handler(const HttpServer& server, const shared_ptr& response, const shared_ptr& request, string RequestType) +{ + + // string operation; + // Document document; + // string url; + // string postcontent; + // if (RequestType == "GET") + // { + // cout << "request path:" << request->path << endl; // url = request->path; // url=UrlDecode(url); @@ -4508,48 +4955,36 @@ void shutdown_handler(const HttpServer& server, const shared_ptr MAX_QUERYLOG_size) - { - //close the old query log file - fclose(query_logfp); - //create and open a new query log file, then save the log name to name.log - string querylog_name = Util::get_date_time(); - int index_space = querylog_name.find(' '); - querylog_name = querylog_name.replace(index_space, 1, 1, '_'); - string namelog_name = QUERYLOG_PATH + NAMELOG_PATH; - FILE *name_logfp = fopen(namelog_name.c_str(), "w"); - fprintf(name_logfp, "%s", querylog_name.c_str()); - - queryLog = QUERYLOG_PATH + querylog_name + ".log"; - //queryLog = querylog_name; - cout << "querLog: " << queryLog << endl; - query_logfp = fopen(queryLog.c_str(), "a"); - if(query_logfp == NULL) - { - cerr << "open query log error"<& response, const shared_ptr& request,string RequestType,string postcontent) { + string remote_ip=getRemoteIp(request); if (!ipCheck(request)) { string error = "IP Blocked!"; - sendResponseMsg(1101, error, response); + sendResponseMsg(1101, error, response,remote_ip,"StopServer"); return false; } @@ -4608,7 +5044,7 @@ const shared_ptr& request,string RequestType,string postcon Document document; string url; - string remote_ip=getRemoteIp(request); + cout<<"request remote ip:"<method << endl; cout << "request http_version:" << request->http_version << endl; @@ -4642,7 +5078,7 @@ const shared_ptr& request,string RequestType,string postcon if (document.HasParseError()) { string error = "the post content is not fit the json format,content=" + strJson; - sendResponseMsg(1003, error, response); + sendResponseMsg(1003, error, response,remote_ip,"StopServer"); return false; } @@ -4664,7 +5100,7 @@ const shared_ptr& request,string RequestType,string postcon else { string msg = "The method type " + request->method + " is not support"; - sendResponseMsg(1000, msg, response); + sendResponseMsg(1000, msg, response,remote_ip,"StopServer"); return false; } @@ -4683,7 +5119,7 @@ const shared_ptr& request,string RequestType,string postcon string checkidentityresult = checkIdentity2(username, password,Util::int2string(server.config.port)); if (checkidentityresult.empty() == false) { - sendResponseMsg(1001, checkidentityresult, response); + sendResponseMsg(1001, checkidentityresult, response,remote_ip,"checkIdentity2"); return false; } cout << "check identity successfully." << endl; @@ -4694,7 +5130,7 @@ const shared_ptr& request,string RequestType,string postcon if (flag == false) { string error= "Stop server failed."; - sendResponseMsg(1005,error,response); + sendResponseMsg(1005,error,response,remote_ip,"StopServer"); return false; } string cmd = "rm system.db/password" + Util::int2string(server.config.port) + ".txt"; @@ -4703,7 +5139,7 @@ const shared_ptr& request,string RequestType,string postcon system(cmd.c_str()); cout<<"Server stopped."<& response, const sha return true; } - - bool addPrivilege(string username, string type, string db_name) { if(username == ROOT_USERNAME) @@ -4809,72 +5243,63 @@ bool addPrivilege(string username, string type, string db_name) std::map::iterator it = users.find(username); if(it != users.end() && db_name != "system") { - pthread_rwlock_unlock(&users_map_lock); if(type == "query") { - pthread_rwlock_wrlock(&(it->second->query_priv_set_lock)); - it->second->query_priv.insert(db_name); string update = "INSERT DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->query_priv_set_lock)); + it->second->query_priv.insert(db_name); pthread_rwlock_unlock(&(it->second->query_priv_set_lock)); - } else if(type == "update") { - pthread_rwlock_wrlock(&(it->second->update_priv_set_lock)); - it->second->update_priv.insert(db_name); string update = "INSERT DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->update_priv_set_lock)); + it->second->update_priv.insert(db_name); pthread_rwlock_unlock(&(it->second->update_priv_set_lock)); - } else if(type == "load") { - pthread_rwlock_wrlock(&(it->second->load_priv_set_lock)); - it->second->load_priv.insert(db_name); string update = "INSERT DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->load_priv_set_lock)); + it->second->load_priv.insert(db_name); pthread_rwlock_unlock(&(it->second->load_priv_set_lock)); } else if(type == "unload") { - pthread_rwlock_wrlock(&(it->second->unload_priv_set_lock)); - it->second->unload_priv.insert(db_name); string update = "INSERT DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->unload_priv_set_lock)); + it->second->unload_priv.insert(db_name); pthread_rwlock_unlock(&(it->second->unload_priv_set_lock)); } else if(type == "restore") { - pthread_rwlock_wrlock(&(it->second->restore_priv_set_lock)); - it->second->restore_priv.insert(db_name); string update = "INSERT DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->restore_priv_set_lock)); + it->second->restore_priv.insert(db_name); pthread_rwlock_unlock(&(it->second->restore_priv_set_lock)); } else if(type == "backup") { - pthread_rwlock_wrlock(&(it->second->backup_priv_set_lock)); - it->second->backup_priv.insert(db_name); string update = "INSERT DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->backup_priv_set_lock)); + it->second->backup_priv.insert(db_name); pthread_rwlock_unlock(&(it->second->backup_priv_set_lock)); } else if(type == "export") { - pthread_rwlock_wrlock(&(it->second->export_priv_set_lock)); - it->second->export_priv.insert(db_name); string update = "INSERT DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->export_priv_set_lock)); + it->second->export_priv.insert(db_name); pthread_rwlock_unlock(&(it->second->export_priv_set_lock)); } + pthread_rwlock_unlock(&users_map_lock); return 1; } else @@ -4883,87 +5308,85 @@ bool addPrivilege(string username, string type, string db_name) return 0; } } + bool delPrivilege(string username, string type, string db_name) { + if (username == ROOT_USERNAME) + { + return 0; + } pthread_rwlock_rdlock(&users_map_lock); std::map::iterator it = users.find(username); - if(it != users.end() && username != ROOT_USERNAME) + bool del_result = false; + if(it != users.end()) { - pthread_rwlock_unlock(&users_map_lock); if(type == "query" && it->second->query_priv.find(db_name) != it->second->query_priv.end()) { - pthread_rwlock_wrlock(&(it->second->query_priv_set_lock)); - it->second->query_priv.erase(db_name); string update = "DELETE DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->query_priv_set_lock)); + it->second->query_priv.erase(db_name); pthread_rwlock_unlock(&(it->second->query_priv_set_lock)); - return 1; + del_result = true; } else if(type == "update" && it->second->update_priv.find(db_name) != it->second->update_priv.end()) { - pthread_rwlock_wrlock(&(it->second->update_priv_set_lock)); - it->second->update_priv.erase(db_name); string update = "DELETE DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->update_priv_set_lock)); + it->second->update_priv.erase(db_name); pthread_rwlock_unlock(&(it->second->update_priv_set_lock)); - return 1; + del_result = true; } else if(type == "load" && it->second->load_priv.find(db_name) != it->second->load_priv.end()) { - pthread_rwlock_wrlock(&(it->second->load_priv_set_lock)); - it->second->load_priv.erase(db_name); string update = "DELETE DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->load_priv_set_lock)); + it->second->load_priv.erase(db_name); pthread_rwlock_unlock(&(it->second->load_priv_set_lock)); - return 1; + del_result = true; } else if(type == "unload" && it->second->unload_priv.find(db_name) != it->second->unload_priv.end()) { - pthread_rwlock_wrlock(&(it->second->unload_priv_set_lock)); - it->second->unload_priv.erase(db_name); string update = "DELETE DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->unload_priv_set_lock)); + it->second->unload_priv.erase(db_name); pthread_rwlock_unlock(&(it->second->unload_priv_set_lock)); - return 1; + del_result = true; } else if(type == "backup" && it->second->backup_priv.find(db_name) != it->second->backup_priv.end()) { - pthread_rwlock_wrlock(&(it->second->backup_priv_set_lock)); - it->second->backup_priv.erase(db_name); string update = "DELETE DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->backup_priv_set_lock)); + it->second->backup_priv.erase(db_name); pthread_rwlock_unlock(&(it->second->backup_priv_set_lock)); - return 1; + del_result = true; } else if(type == "restore" && it->second->restore_priv.find(db_name) != it->second->restore_priv.end()) { - pthread_rwlock_wrlock(&(it->second->restore_priv_set_lock)); - it->second->restore_priv.erase(db_name); string update = "DELETE DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->restore_priv_set_lock)); + it->second->restore_priv.erase(db_name); pthread_rwlock_unlock(&(it->second->restore_priv_set_lock)); - return 1; + del_result = true; } else if(type == "export" && it->second->export_priv.find(db_name) != it->second->export_priv.end()) { - pthread_rwlock_wrlock(&(it->second->export_priv_set_lock)); - it->second->export_priv.erase(db_name); string update = "DELETE DATA {<" + username + "> <" + db_name + ">.}"; updateSys(update); - + pthread_rwlock_wrlock(&(it->second->export_priv_set_lock)); + it->second->export_priv.erase(db_name); pthread_rwlock_unlock(&(it->second->export_priv_set_lock)); - return 1; + del_result = true; } } pthread_rwlock_unlock(&users_map_lock); - return 0; + return del_result; } @@ -4974,55 +5397,57 @@ int clearPrivilege(string username) string update=""; if(it != users.end() && username != ROOT_USERNAME) { - pthread_rwlock_unlock(&users_map_lock); update = "DELETE where {<" + username + "> ?x.}"; + updateSys(update); pthread_rwlock_wrlock(&(it->second->query_priv_set_lock)); it->second->query_priv.clear(); pthread_rwlock_unlock(&(it->second->query_priv_set_lock)); - - - updateSys(update); + update = "DELETE where {<" + username + "> ?x.}"; updateSys(update); pthread_rwlock_wrlock(&(it->second->load_priv_set_lock)); it->second->load_priv.clear(); pthread_rwlock_unlock(&(it->second->load_priv_set_lock)); + update = "DELETE where {<" + username + "> ?x.}"; updateSys(update); pthread_rwlock_wrlock(&(it->second->unload_priv_set_lock)); it->second->unload_priv.clear(); pthread_rwlock_unlock(&(it->second->unload_priv_set_lock)); + update = "DELETE where {<" + username + "> ?x.}"; updateSys(update); pthread_rwlock_wrlock(&(it->second->update_priv_set_lock)); it->second->update_priv.clear(); pthread_rwlock_unlock(&(it->second->update_priv_set_lock)); + update = "DELETE where {<" + username + "> ?x.}"; updateSys(update); pthread_rwlock_wrlock(&(it->second->backup_priv_set_lock)); it->second->backup_priv.clear(); pthread_rwlock_unlock(&(it->second->backup_priv_set_lock)); + update = "DELETE where {<" + username + "> ?x.}"; updateSys(update); pthread_rwlock_wrlock(&(it->second->restore_priv_set_lock)); it->second->restore_priv.clear(); pthread_rwlock_unlock(&(it->second->restore_priv_set_lock)); + update = "DELETE where {<" + username + "> ?x.}"; updateSys(update); pthread_rwlock_wrlock(&(it->second->export_priv_set_lock)); it->second->export_priv.clear(); pthread_rwlock_unlock(&(it->second->export_priv_set_lock)); + sysrefresh(); pthread_rwlock_unlock(&users_map_lock); return 1; - - } else { pthread_rwlock_unlock(&users_map_lock); - return -1; + return -1; } @@ -5034,28 +5459,31 @@ bool checkPrivilege(string username, string type, string db_name) if(username == ROOT_USERNAME) return 1; + if (type == "login" || type == "testConnect" || type == "getCoreVersion" + || type == "funquery" || type == "funcudb" || type == "funreview" + || type == "check" || type == "show") + { + return 1; + } pthread_rwlock_rdlock(&users_map_lock); std::map::iterator it = users.find(username); - //pthread_rwlock_unlock(&users_map_lock); - if(type == "query") + int check_result = 0; + if(type == "query" || type == "monitor") { pthread_rwlock_rdlock(&(it->second->query_priv_set_lock)); if(it->second->query_priv.find(db_name) != it->second->query_priv.end()) { - pthread_rwlock_unlock(&(it->second->query_priv_set_lock)); - pthread_rwlock_unlock(&users_map_lock); - return 1; + check_result = 1; } pthread_rwlock_unlock(&(it->second->query_priv_set_lock)); } - else if(type == "update") + else if(type == "update" || type == "batchInsert" || type == "batchRemove" + || type == "begin" || type == "tquery" || type == "commit" || type == "rollback") { pthread_rwlock_rdlock(&(it->second->update_priv_set_lock)); if(it->second->update_priv.find(db_name) != it->second->update_priv.end()) { - pthread_rwlock_unlock(&(it->second->update_priv_set_lock)); - pthread_rwlock_unlock(&users_map_lock); - return 1; + check_result = 1; } pthread_rwlock_unlock(&(it->second->update_priv_set_lock)); } @@ -5064,9 +5492,7 @@ bool checkPrivilege(string username, string type, string db_name) pthread_rwlock_rdlock(&(it->second->load_priv_set_lock)); if(it->second->load_priv.find(db_name) != it->second->load_priv.end()) { - pthread_rwlock_unlock(&(it->second->load_priv_set_lock)); - pthread_rwlock_unlock(&users_map_lock); - return 1; + check_result = 1; } pthread_rwlock_unlock(&(it->second->load_priv_set_lock)); } @@ -5075,9 +5501,7 @@ bool checkPrivilege(string username, string type, string db_name) pthread_rwlock_rdlock(&(it->second->unload_priv_set_lock)); if(it->second->unload_priv.find(db_name) != it->second->unload_priv.end()) { - pthread_rwlock_unlock(&(it->second->unload_priv_set_lock)); - pthread_rwlock_unlock(&users_map_lock); - return 1; + check_result = 1; } pthread_rwlock_unlock(&(it->second->unload_priv_set_lock)); } @@ -5086,9 +5510,7 @@ bool checkPrivilege(string username, string type, string db_name) pthread_rwlock_rdlock(&(it->second->restore_priv_set_lock)); if(it->second->restore_priv.find(db_name) != it->second->restore_priv.end()) { - pthread_rwlock_unlock(&(it->second->restore_priv_set_lock)); - pthread_rwlock_unlock(&users_map_lock); - return 1; + check_result = 1; } pthread_rwlock_unlock(&(it->second->restore_priv_set_lock)); } @@ -5097,9 +5519,7 @@ bool checkPrivilege(string username, string type, string db_name) pthread_rwlock_rdlock(&(it->second->backup_priv_set_lock)); if(it->second->backup_priv.find(db_name) != it->second->backup_priv.end()) { - pthread_rwlock_unlock(&(it->second->backup_priv_set_lock)); - pthread_rwlock_unlock(&users_map_lock); - return 1; + check_result = 1; } pthread_rwlock_unlock(&(it->second->backup_priv_set_lock)); } @@ -5108,14 +5528,13 @@ bool checkPrivilege(string username, string type, string db_name) pthread_rwlock_rdlock(&(it->second->export_priv_set_lock)); if(it->second->export_priv.find(db_name) != it->second->export_priv.end()) { - pthread_rwlock_unlock(&(it->second->export_priv_set_lock)); - pthread_rwlock_unlock(&users_map_lock); - return 1; + check_result = 1; } pthread_rwlock_unlock(&(it->second->export_priv_set_lock)); } + cout << "check [" << username << "] [" << db_name << "] ["<< type << "] privilege: " <& response, string file_name, int page_no, int page_size) +void quereyLog_thread_new(const shared_ptr& response, string file_name, int page_no, int page_size,string ip) { + string operation="queryLog"; query_log_lock.lock(); int totalSize = 0; int totalPage = 0; @@ -5620,23 +6040,41 @@ void quereyLog_thread_new(const shared_ptr& response, stri startLine = (page_no - 1)*page_size + 1; endLine = page_no*page_size + 1; //count total - while (getline(in, line)) + while (getline(in, line, '\n')) { totalSize++; } in.close(); - cout<< "totalSize=" << totalSize << ", page_no=" << page_no << ", page_size=" << page_size << endl; + totalPage = (totalSize/page_size) + (totalSize%page_size == 0 ? 0 : 1); + if (page_no > totalPage) + { + throw runtime_error("page_no more then max_page_no " + totalPage); + } + startLine = totalSize - page_size*page_no + 1; + endLine = totalSize - page_size*(page_no - 1) + 1; + if (startLine < 1) + { + startLine = 1; + } + cout<< "startLine=" << startLine << ", endLine=" << endLine << ", totalSize=" << totalSize << ", totalPage=" << totalPage << endl; // seek to start line; in.open(queryLog, ios::in); seek_to_line(in, startLine); - bool success = true; + vector lines; + while (startLine < endLine && getline(in, line, '\n')) { + lines.push_back(line); + startLine++; + } + in.close(); stringstream str_stream; + size_t count; str_stream << "["; - while (startLine < endLine && getline(in, line)) { + count = lines.size(); + for (size_t i = 0; i < count; i++) + { + line = lines[count - i - 1]; str_stream << line; - startLine++; } - in.close(); line = str_stream.str(); line = Util::string_replace(line, "\n", ""); if (line[line.length() - 1] == ',') @@ -5653,16 +6091,17 @@ void quereyLog_thread_new(const shared_ptr& response, stri all.AddMember("pageNo", page_no, all.GetAllocator()); all.AddMember("pageSize", page_size, all.GetAllocator()); all.AddMember("list", list, all.GetAllocator()); + } catch (std::exception &e) { all.AddMember("StatusCode", 1005, all.GetAllocator()); all.AddMember("message", "Error! Query log corrupted", all.GetAllocator()); + writeIpAccessLog(ip,operation,"get data error!",1005); } } else { - list.Parse("[]"); all.AddMember("StatusCode", 0, all.GetAllocator()); all.AddMember("StatusMsg", "Get query log success", all.GetAllocator()); all.AddMember("totalSize", totalSize, all.GetAllocator()); @@ -5677,5 +6116,240 @@ void quereyLog_thread_new(const shared_ptr& response, stri string resJson = buffer.GetString(); *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; query_log_lock.unlock(); + writeIpAccessLog(ip,operation,"get data successfully!",0); + return; +} + +/** + * access log thread + * + * @param response + * @param file_name + * @param page_no + * @param page_size + */ +void accessLog_thread_new(const shared_ptr& response, string file_name, int page_no, int page_size,string ip) +{ + string operation="accessLog"; + ip_log_lock.lock(); + int totalSize = 0; + int totalPage = 0; + string queryLog = IP_ACCESS_PATH + file_name + ".log"; + Document all; + Document list; + all.SetObject(); + list.SetArray(); + if(Util::file_exist(queryLog)) + { + try + { + ifstream in; + string line; + in.open(queryLog, ios::in); + // start paging + int startLine; //include + int endLine; //exclude + if (page_no < 1) + { + page_no = 1; + } + if (page_size < 1) + { + page_size = 10; + } + // count total + while (getline(in, line, '\n')) + { + totalSize++; + } + in.close(); + totalPage = (totalSize/page_size) + (totalSize%page_size == 0 ? 0 : 1); + if (page_no > totalPage) + { + throw runtime_error("page_no more then max_page_no " + totalPage); + } + startLine = totalSize - page_size*page_no + 1; + endLine = totalSize - page_size*(page_no - 1) + 1; + if (startLine < 1) + { + startLine = 1; + } + cout<< "startLine=" << startLine << ", endLine=" << endLine << ", totalSize=" << totalSize << ", totalPage=" << totalPage << endl; + // seek to start line; + in.open(queryLog, ios::in); + seek_to_line(in, startLine); + vector lines; + while (startLine < endLine && getline(in, line, '\n')) { + lines.push_back(line); + startLine++; + } + in.close(); + stringstream str_stream; + size_t count; + str_stream << "["; + count = lines.size(); + for (size_t i = 0; i < count; i++) + { + line = lines[count - i - 1]; + str_stream << line; + } + line = str_stream.str(); + line = Util::string_replace(line, "\n", ""); + if (line[line.length() - 1] == ',') + { + line = line.substr(0, (line.length() - 1)); + } + line.push_back(']'); + list.Parse(line.c_str()); + + all.AddMember("StatusCode", 0, all.GetAllocator()); + all.AddMember("StatusMsg", "Get access log success", all.GetAllocator()); + all.AddMember("totalSize", totalSize, all.GetAllocator()); + all.AddMember("totalPage", totalPage, all.GetAllocator()); + all.AddMember("pageNo", page_no, all.GetAllocator()); + all.AddMember("pageSize", page_size, all.GetAllocator()); + all.AddMember("list", list, all.GetAllocator()); + + } + catch (std::exception &e) + { + all.AddMember("StatusCode", 1005, all.GetAllocator()); + all.AddMember("message", "Error! Access log corrupted", all.GetAllocator()); + writeIpAccessLog(ip,operation,"get data error!",1005); + } + } + else + { + all.AddMember("StatusCode", 0, all.GetAllocator()); + all.AddMember("StatusMsg", "Get access log success", all.GetAllocator()); + all.AddMember("totalSize", totalSize, all.GetAllocator()); + all.AddMember("totalPage", totalPage, all.GetAllocator()); + all.AddMember("pageNo", page_no, all.GetAllocator()); + all.AddMember("pageSize", page_size, all.GetAllocator()); + all.AddMember("list", list, all.GetAllocator()); + } + StringBuffer buffer; + Writer writer(buffer); + all.Accept(writer); + string resJson = buffer.GetString(); + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; + ip_log_lock.unlock(); + // writeIpAccessLog(ip,operation,"get data successfully!",0); return; +} + +/** + * @brief IP manager + * + * @param response + * @param ips ip string + * @param ip_type 1-black ip 2-white ip + * @param type 1-query 2-save + */ +void IPManager_thread_new(const shared_ptr& response, string ips, string ip_type, string type) +{ + Document all; + Document::AllocatorType& allocator = all.GetAllocator(); + all.SetObject(); + if ( type == "1") // query + { + Document responseBody; + Document listDoc; + responseBody.SetObject(); + listDoc.SetArray(); + if (whiteList) // white IP + { + cout << "IP white List enabled." << endl; + responseBody.AddMember("ip_type", "2", allocator); + for (std::set::iterator it = ipWhiteList->ipList.begin(); it!=ipWhiteList->ipList.end();it++) + { + Value item(kStringType); + item.SetString((*it).c_str(), allocator); + listDoc.PushBack(item, allocator); + } + } + else if (blackList) // black IP + { + cout << "IP black List enabled." << endl; + responseBody.AddMember("ip_type", "1", allocator); + for (std::set::iterator it = ipBlackList->ipList.begin(); it!=ipBlackList->ipList.end();it++) + { + Value item(kStringType); + item.SetString((*it).c_str(), allocator); + listDoc.PushBack(item, allocator); + } + } + else + { + string resJson = CreateJson(1005, "please configure ip_deny_path or ip_allow_path in the conf.ini file first.", 0); + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; + return; + } + responseBody.AddMember("ips", listDoc, allocator); + + all.AddMember("StatusCode", 0, allocator); + all.AddMember("StatusMsg", "success", allocator); + all.AddMember("ResponseBody", responseBody, allocator); + StringBuffer buffer; + Writer writer(buffer); + all.Accept(writer); + string resJson = buffer.GetString(); + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; + return; + } + else if (type == "2") // save + { + if (ips.empty()) + { + string resJson = CreateJson(1003, "the ips can't be empty", 0); + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; + return; + } + vector ipVector; + Util::split(ips,",", ipVector); + if (ip_type == "1") // black IP + { + if (blackList) + { + ipBlackList->UpdateIPToFile(ipBlackFile, ipVector, "update by wrokbanch"); + // realod ip list + ipBlackList->Load(ipBlackFile); + } + else + { + string resJson = CreateJson(1005, "ip_deny_path is not configured, please configure it in the conf.ini file first.", 0); + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; + return; + } + } + else if (ip_type == "2") // white IP + { + if (whiteList) + { + ipWhiteList->UpdateIPToFile(ipWhiteFile, ipVector, "update by wrokbanch"); + // realod ip list + ipWhiteList->Load(ipWhiteFile); + } + else + { + string resJson = CreateJson(1005, "ip_allow_path is not configured, please configure it in the conf.ini file first.", 0); + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; + return; + } + } + else + { + string resJson = CreateJson(1003, "ip_type is invalid, please look up the api document.", 0); + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; + return; + } + all.AddMember("StatusCode", 0, allocator); + all.AddMember("StatusMsg", "success", allocator); + StringBuffer buffer; + Writer writer(buffer); + all.Accept(writer); + string resJson = buffer.GetString(); + *response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson; + return; + } } \ No newline at end of file diff --git a/Main/ginit.cpp b/Main/ginit.cpp index 7c02f326..274b153f 100644 --- a/Main/ginit.cpp +++ b/Main/ginit.cpp @@ -53,7 +53,9 @@ int main(int argc, char * argv[]) Util::init_backuplog(); cout<<"init backuplog successfully!"<") { cout<<"The database does not exist."<") + if (ask_rs.answer[0][0] == "false") { cout << "The database does not exist. Rebuild" << endl; string time = Util::get_backup_time(backup_path, db_name); diff --git a/Parser/QueryParser.cpp b/Parser/QueryParser.cpp index 2eb4d356..507d5f12 100644 --- a/Parser/QueryParser.cpp +++ b/Parser/QueryParser.cpp @@ -1247,8 +1247,8 @@ antlrcpp::Any QueryParser::visitBind(SPARQLParser::BindContext *ctx, \ unaryexpression(0)->primaryexpression()->rDFLiteral(); if (!rdflCtx) throw runtime_error("[ERROR] Currently BIND only supports assigning a string to a var."); - antlr4::tree::ParseTree *curr = ctx; - for (int i = 0; i < 10; i++) + antlr4::tree::ParseTree *curr = ctx->expression(); + for (int i = 0; i < 9; i++) { // Make sure only one children along the way if (curr->children.size() > 1) diff --git a/Query/GeneralEvaluation.cpp b/Query/GeneralEvaluation.cpp index 611dee74..0db88938 100644 --- a/Query/GeneralEvaluation.cpp +++ b/Query/GeneralEvaluation.cpp @@ -1330,7 +1330,14 @@ void GeneralEvaluation::getFinalResult(ResultSet &ret_result) if (!proj[0].path_args.pred_set.empty()) { for (auto pred : proj[0].path_args.pred_set) - pred_id_set.push_back(kvstore->getIDByPredicate(pred)); + { + TYPE_PREDICATE_ID pred_id = kvstore->getIDByPredicate(pred); + // cout << "pred_set id:" << pred_id << endl; + if (pred_id != INVALID) + { + pred_id_set.push_back(pred_id); + } + } } else { @@ -1494,6 +1501,7 @@ void GeneralEvaluation::getFinalResult(ResultSet &ret_result) break; } ss << "]}\""; + cout << "paths length:" << uid_ls.size() * vid_ls.size() << endl; if (proj[0].aggregate_type == QueryTree::ProjectionVar::cycleBoolean_type) { if (exist) @@ -1502,7 +1510,9 @@ void GeneralEvaluation::getFinalResult(ResultSet &ret_result) new_result0.result.back().str[proj2new[0] - new_result0_id_cols] = "\"false\"^^"; } else - ss >> new_result0.result.back().str[proj2new[0] - new_result0_id_cols]; + { + new_result0.result.back().str[proj2new[0] - new_result0_id_cols] = ss.str(); + } } else { @@ -2273,7 +2283,14 @@ void GeneralEvaluation::getFinalResult(ResultSet &ret_result) if (!proj[i].path_args.pred_set.empty()) { for (auto pred : proj[i].path_args.pred_set) - pred_id_set.push_back(kvstore->getIDByPredicate(pred)); + { + TYPE_PREDICATE_ID pred_id = kvstore->getIDByPredicate(pred); + // cout << "pred_set id:" << pred_id << endl; + if (pred_id != INVALID) + { + pred_id_set.push_back(pred_id); + } + } } else { @@ -2435,6 +2452,7 @@ void GeneralEvaluation::getFinalResult(ResultSet &ret_result) break; } ss << "]}\""; + cout << "paths length:" << uid_ls.size() * vid_ls.size() << endl; if (proj[i].aggregate_type == QueryTree::ProjectionVar::cycleBoolean_type) { if (exist) @@ -2443,7 +2461,9 @@ void GeneralEvaluation::getFinalResult(ResultSet &ret_result) new_result0.result.back().str[proj2new[i] - new_result0_id_cols] = "\"false\"^^"; } else - ss >> new_result0.result.back().str[proj2new[i] - new_result0_id_cols]; + { + new_result0.result.back().str[proj2new[i] - new_result0_id_cols] = ss.str(); + } } } begin = end + 1; diff --git a/README.md b/README.md index 54f35f81..04cfb7dc 100644 --- a/README.md +++ b/README.md @@ -1,100 +1,15 @@ -
+# gStore System -# gStore- Native graph database system for large-scale knowledge graph application +Gstore System(also called gStore) is a graph database engine for managing large graph-structured data, which is open-source and targets at Linux operation systems. The whole project is written in C++, with the help of some libraries such as readline, antlr, and so on. Only source tarballs are provided currently, which means you have to compile the source code if you want to use our system. -**The latest version:**  - -The latest version is 0.9.1, updated on November 25, 2021. - -**Help document:**  - -[gStore帮助手册-0.9.1版本-中文版](http://file.gstore.cn/f/50f350b7bac047688dab/?dl=1) - -[gStore User Guide - 0.9.1 Version-English](http://file.gstore.cn/f/d69814aabd2e40fead22/?dl=1) - -**Homepage:** - -[ http://gstore.cn](http://gstore.cn) - -**Product trial:** - -[http://cloud.gstore.cn](http://cloud.gstore.cn) - -**Discussion group:** - -[http://www.gstore.cn/pcsite/index.html#/medical（only chinese）](http://cloud.gstore.cn) - -[https://github.com/pkumod/gStore/issues](https://github.com/pkumod/gStore/issues) - -**Development team:** - -Data Management Laboratory (PKUMOD) of Wangxuan Institute of Computer Technology of Peking University. - -**License:** - -[ BSD-3-Clause License](https://github.com/pkumod/gStore/blob/0.9.1/LICENSE) - -**Contact us:** - -Technical problems: gstore@pku.edu.cn , service@gstore.cn - -Business issues:gstore@pku.edu.cn , jiuhua.qi@cqbdri.pku.edu.cn - - - -## Introduction - -gStore is a graph-based RDF data management system(or what is commonly called a "triple store") that maintains the graph structure of the original [RDF](http://www.w3.org/TR/rdf11-concepts/) data. Its data model is a labeled, directed multi edge graph, where each vertex corresponds to a subject or an object. - -We represent a given [SPARQL](http://www.w3.org/TR/sparql11-overview/) query by a query graph Q. Query processing involves finding subgraph matches of Q over the RDF graph G, instead of joining tables in relational data management system. gStore incorporates an index over the RDF graph (called VS-tree) to speed up query processing. VS-tree is a height balanced tree with a number of associated pruning techniques to speed up subgraph matching. - -**(NOTICE: Homomorphism is used here, instead of isomorphism)** - - - -## Fature - -The important features of gStore are as follows: - -- gStore manages RDF repository from a graph database perspective. -- Support SPARQL 1.1 standard query language (including operation primitives such as UNION, FILTER, OPTIONAL and add, delete and modify). -- gStore supports both query and update efficiently. -- gStore can handle, in a uniform manner, different data types (strings and numerical data) and SPARQL queries with aggregate, range operators. -- Support distributed deployment(Paid support). -- The standalone version supports 5 billion triples. -- The system has independent backup and error recovery functions, providing high reliable performance guarantee. -- Complete support for transaction ACID. - - - -## Publication - -The first essay to come up with Gstore System is [gStore_VLDB](http://file.gstore.cn/f/f8dcf8d1476b4982a182/). - -- Lei Zou, Jinghui Mo, Lei Chen,M. Tamer Özsu, Dongyan Zhao, [gStore: Answering SPARQL Queries Via Subgraph Matching](http://www.icst.pku.edu.cn/intro/leizou/projects/papers/p482-zou.pdf), Proc. VLDB 4(8): 482-493, 2011. -- Lei Zou, M. Tamer Özsu,Lei Chen, Xuchuan Shen, Ruizhe Huang, Dongyan Zhao, [gStore: A Graph-based SPARQL Query Engine](http://www.icst.pku.edu.cn/intro/leizou/projects/papers/gStoreVLDBJ.pdf), VLDB Journal , 23(4): 565-590, 2014. -- Xuchuan Shen, Lei Zou, M. Tamer Özsu, Lei Chen, Youhuan Li, Shuo Han, Dongyan Zhao, [A Graph-based RDF Triple Store](http://www.icst.pku.edu.cn/intro/leizou/projects/papers/demo.pdf), in Proc. 31st International Conference on Data Engineering (ICDE), 2015; To appear (demo). -- Dong Wang, Lei Zou, Yansong Feng, Xuchuan Shen, Jilei Tian, and Dongyan Zhao, [S-store: An Engine for Large RDF Graph Integrating Spatial Information](http://www.icst.pku.edu.cn/intro/leizou/projects/papers/Store.pdf), in Proc. 18th International Conference on Database Systems for Advanced Applications (DASFAA), pages 31-47, 2013. -- Dong Wang, Lei Zou and Dongyan Zhao, [gst-Store: An Engine for Large RDF Graph Integrating Spatiotemporal Information](http://www.icst.pku.edu.cn/intro/leizou/projects/papers/edbtdemo2014.pdf), in Proc. 17th International Conference on Extending Database Technology (EDBT), pages 652-655, 2014 (demo). -- Lei Zou, Yueguo Chen, [A Survey of Large-Scale RDF Data Management](http://www.icst.pku.edu.cn/intro/leizou/documentation/pdf/2012CCCF.pdf), Comunications of CCCF Vol.8(11): 32-43, 2012 (Invited Paper, in Chinese). +**The formal help document is in [English(EN)](docs/help/gStore_help.pdf) and [中文(ZH)](docs/help/gStore_help_ZH.pdf).** +**The formal experiment result is in [Experiment](docs/test/formal_experiment.pdf).** +**We have built an IRC channel named #gStore on freenode, and you can visit [the homepage of gStore](http://gstore.cn).** ## Change log -**0.9.1（beta）：2021-11-25** - -New features in gStore 0.9.1 are listed as follows: - -- Decoupling the parsing and execution of queries in kernel, and further improvements on the query performance through optimized join ordering and other techniques. On complex queries, the performance is improved by over 40%. -- Rewriting of the HTTP service component, ghttp, with improved robustness and the addition of functions such as user permission, heartbeat detection, batch import, and batch deletion; API documents are added. -- Implementation of the Personalized PageRank (PPR) extension function, which can be invoked in the SELECT clause to calculate the correlation between entities. -- Support for arithmetic operations (e.g., `?x + ?y = 5`) in the FILTER clause. -- Support for transactional operations, such as begin, tquery (transactional query), commit, and rollback; -- A new executive component, gserver, is added to provide another pathway for remote access of gStore aside from the ghttp component, which implements two-way communication via the socket API. -- Unification of the format of command line arguments of executive components. The `--help` option is uniformly introduced (e.g., `$ bin/gbuild --help` or `$ bin/gbuild -h`), by which users can view the command manual including the meaning of each option. -- A number of bug fixes. - **0.9（beta）：2021-02-10** New features in version 0.9 include: @@ -111,7 +26,6 @@ New features in version 0.9 include: - Expanding data structures to hold large-scale graphs of up to five billion triples. The version is a beta version, you can get it by : - ``` git clone https://github.com/pkumod/gStore.git ``` @@ -119,18 +33,15 @@ git clone https://github.com/pkumod/gStore.git **0.8（Stable）** The version is a stable version ,you can get it by - ``` git clone -b 0.8 https://github.com/pkumod/gStore.git ``` - - + ## Getting Started - -### **Compile from Source** +### Compile from Source This system is really user-friendly and you can pick it up in several minutes. Remember to check your platform where you want to run this system by viewing [System Requirements](docs/DEMAND.md). After all are verified, please get this project's source code. There are several ways to do this: - (suggested)type `git clone https://github.com/pkumod/gStore.git` in your terminal or use git GUI to acquire it @@ -160,59 +71,42 @@ To run gStore, please type `bin/gbuild database_name dataset_path` to build a da - - - - - -## Performance - -**The formal experiment report is in [EXPERIMENT](test/formal_experiment.pdf).** - -## Preparation - -We have compared the performance of gStore with several other database systems, such as [Jena](http://jena.apache.org/), [Sesame](http://www.rdf4j.org/), [Virtuoso](http://virtuoso.openlinksw.com/) and so on. Contents to be compared are the time to build database, the size of the built database, the time to answer single SPARQL query and the matching case of single query's results. In addition, if the memory cost is very large(>20G), we will record the memory cost when running these database systems.(not accurate, just for your reference) - -To ensure all database systems can run correctly on all datasets and queries, the format of datasets must be supported by all database systems and the queries should not contain update operations, aggregate operations and operations related with uncertain predicates. Notice that when measuring the time to answer queries, the time of loading database index should not be included. To ensure this principle, we load the database index first for some database systems, and warm up several times for others. - -Datasets used here are WatDiv, Lubm, Bsbm and DBpedia. Some of them are provided by websites, and others are generated by algorithms. Queries are generated by algorithms or written by us. - -The experiment environment is a CentOS server, whose memory size is 82G and disk size is 7T. We use [full_test](../test/full_test.sh) to do this test. - -## Result - -This program produces many logs placed in result.log/, load.log/ and time.log/. You can see that all results of all queries are matched by viewing files in result.log/, and the time cost and space cost of gStore to build database are larger than others by viewing files in load.log/. More precisely, there is an order of magnitude difference between gStore and others in the time/space cost of building database. - -Through analysing time.log/, we can find that gStore behave better than others on very complicated queries(many variables, circles, etc). For other simple queries, there is not much difference between the time of these database systems. - -Generally speaking, the memory cost of gStore when answering queries is higher than others. More complicated the query is and more large the dataset is, more apparent the phenomenon is. - -You can find more detailed information in [test report](pdf/gstore_test_report.pdf). Notice that some questions in the test report have already be solved now. - - - ## Advanced Help If you want to understand the details of the gStore system, or you want to try some advanced operations(for example, using the API, server/client), please see the chapters below. +- [Basic Introduction](docs/INTRO.md): introduce the theory and features of gStore + +- [Install Guide](docs/INSTALL.md): instructions on how to install this system + - [How To Use](docs/USAGE.md): detailed information about using the gStore system - [API Explanation](docs/API.md): guide you to develop applications based on our API - [Project Structure](docs/STRUCT.md): show the whole structure and process of this project -- - - +- [Related Essays](docs/ESSAY.md): contain essays and publications related with gStore - +- [Update Logs](docs/CHANGELOG.md): keep the logs of the system updates -## Submit questions +- [Test Results](docs/TEST.md): present the test results of a series of experiments -Bugs are recorded in [BUG REPORT](docs/BUGS.md). +- - - + +## Other Business -You are welcome to submit any advice or errors in the Github Issues part of this repository and [official website forum](http://www.gstore.cn/pcsite/index.html#/medical), if not requiring in-time reply. However, if you want to urgent on us to deal with your reports, please email to , to submit your suggestions and report bugs. A full list of our whole team is in [Mailing List](docs/MAIL.md). +Bugs are recorded in [BUG REPORT](docs/BUGS.md). +You are welcomed to submit the bugs you discover if they do not exist in this file. We have written a series of short essays addressing recurring challenges in using gStore to realize applications, which are placed in [Recipe Book](docs/TIPS.md). +You are welcome to report any advice or errors in the github Issues part of this repository, if not requiring in-time reply. However, if you want to urgent on us to deal with your reports, please email to to submit your suggestions and report bugs. A full list of our whole team is in [Mailing List](docs/MAIL.md). + +There are some restrictions when you use the current gStore project, you can see them on [Limit Description](docs/LIMIT.md). + Sometimes you may find some strange phenomena(but not wrong case), or something hard to understand/solve(don't know how to do next), then do not hesitate to visit the [Frequently Asked Questions](docs/FAQ.md) page. -Graph database engine is a new area and we are still trying to go further, and we hope more and more people will support or even join us. You can support in many ways: +Graph database engine is a new area and we are still trying to go further. Things we plan to do next is in [Future Plan](docs/PLAN.md) chapter, and we hope more and more people will support or even join us. You can support in many ways: - watch/star our project @@ -224,3 +118,7 @@ Graph database engine is a new area and we are still trying to go further, and w People who inspire us or contribute to this project will be listed in the [Thanks List](docs/THANK.md) chapter. + + + + diff --git a/Util/INIParser.cpp b/Util/INIParser.cpp index cc68979b..782c1021 100644 --- a/Util/INIParser.cpp +++ b/Util/INIParser.cpp @@ -11,6 +11,7 @@ string & TrimString(string & str) //read in INI file and parse it int INIParser::ReadINI(string path) { + cout<<"path："<::iterator itr = map_tmp.begin(); itr != map_tmp.end(); ++itr) { - //cout << itr->first << endl; + cout << itr->first << endl; for (vector::iterator sub_itr = vec_ini.begin(); sub_itr != vec_ini.end(); ++sub_itr) { if (sub_itr->root == itr->first) { - //cout << sub_itr->key << "=" << sub_itr->value << endl; + cout << sub_itr->key << "=" << sub_itr->value << endl; sn.InsertElement(sub_itr->key, sub_itr->value); } } diff --git a/Util/IPBlackList.cpp b/Util/IPBlackList.cpp index e7c7a245..866e1c39 100644 --- a/Util/IPBlackList.cpp +++ b/Util/IPBlackList.cpp @@ -86,3 +86,38 @@ bool IPBlackList::Check(std::string ip){ } return true; } + +void IPBlackList::InsertIPToFile(std::string file, std::string ip, std::string reason) +{ + ofstream outfile; + // ios::app指从文件尾开始输出 + outfile.open(file, ios::out | ios::app); + if (!outfile) + { + cout << "open white list file failed." << endl; + return; + } + + //使输出格式为浮点 + outfile << "#" << reason << "\n"; //以空格为间隔写 x y + outfile << ip << "\n"; //一行写完换行 + outfile.close(); + this->ipList.insert(ip); +} + +void IPBlackList::UpdateIPToFile(std::string file, vector& ips, std::string reason) +{ + ofstream outfile; + outfile.open(file.c_str()); + if (!outfile) + { + cout << "open black list file failed." << endl; + return; + } + outfile << "#" << reason << "\n"; + for(vector::iterator it = ips.begin(); it != ips.end(); it++) + { + outfile << (*it) << "\n"; + } + outfile.close(); +} \ No newline at end of file diff --git a/Util/IPBlackList.h b/Util/IPBlackList.h index a9b0e353..596bdb65 100644 --- a/Util/IPBlackList.h +++ b/Util/IPBlackList.h @@ -1,3 +1,11 @@ +/* + * @Author: your name + * @Date: 2021-12-10 20:34:32 + * @LastEditTime: 2021-12-14 15:55:47 + * @LastEditors: Please set LastEditors + * @Description: 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE + * @FilePath: /gstore/Util/IPBlackList.h + */ /*============================================================================= # Filename: IPBlackList.h # Author: Jing Li @@ -20,5 +28,7 @@ public: void ReadIPFromFile(std::string file); bool Check(std::string ip); //bool AddIP(std::string ip); - + void InsertIPToFile(std::string file,std::string ip,std::string reason); + void UpdateIPToFile(std::string file, vector& ips, std::string reason); + }; \ No newline at end of file diff --git a/Util/IPWhiteList.cpp b/Util/IPWhiteList.cpp index 74487b8b..84b6680e 100644 --- a/Util/IPWhiteList.cpp +++ b/Util/IPWhiteList.cpp @@ -83,4 +83,39 @@ bool IPWhiteList::Check(std::string ip){ } } return false; +} + +void IPWhiteList::InsertIPToFile(std::string file, std::string ip, std::string reason) +{ + ofstream outfile; + // ios::app指从文件尾开始输出 + outfile.open(file, ios::out | ios::app); + if (!outfile) + { + cout << "open white list file failed." << endl; + return; + } + + //使输出格式为浮点 + outfile << "#" << reason << "\n"; //以空格为间隔写 x y + outfile << ip << "\n"; //一行写完换行 + outfile.close(); + this->ipList.insert(ip); +} + +void IPWhiteList::UpdateIPToFile(std::string file, vector& ips, std::string reason) +{ + ofstream outfile; + outfile.open(file.c_str()); + if (!outfile) + { + cout << "open white list file failed." << endl; + return; + } + outfile << "#" << reason << "\n"; + for(vector::iterator it = ips.begin(); it != ips.end(); it++) + { + outfile << (*it) << "\n"; + } + outfile.close(); } \ No newline at end of file diff --git a/Util/IPWhiteList.h b/Util/IPWhiteList.h index 2f23d9ac..45654647 100644 --- a/Util/IPWhiteList.h +++ b/Util/IPWhiteList.h @@ -1,3 +1,11 @@ +/* + * @Author: your name + * @Date: 2021-12-10 20:45:51 + * @LastEditTime: 2021-12-14 15:56:14 + * @LastEditors: Please set LastEditors + * @Description: 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE + * @FilePath: /gstore/Util/IPWhiteList.h + */ /*============================================================================= # Filename: IPWhiteList.h # Author: Jing Li @@ -20,6 +28,8 @@ public: void Load(std::string file); void ReadIPFromFile(std::string file); bool Check(std::string ip); + void InsertIPToFile(std::string file,std::string ip,std::string reason); //bool AddIP(std::string ip); + void UpdateIPToFile(std::string file, vector& ips, std::string reason); }; \ No newline at end of file diff --git a/Util/Latch.cpp b/Util/Latch.cpp index e2633efa..9e982ad8 100644 --- a/Util/Latch.cpp +++ b/Util/Latch.cpp @@ -1,3 +1,11 @@ +/* + * @Author: your name + * @Date: 2022-03-28 09:14:43 + * @LastEditTime: 2022-03-28 09:14:43 + * @LastEditors: your name + * @Description: 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE + * @FilePath: /gstore/Util/Latch.cpp + */ #include "Latch.h" Latch::Latch() diff --git a/Util/Util.cpp b/Util/Util.cpp index c466e8a0..7af1d5b8 100644 --- a/Util/Util.cpp +++ b/Util/Util.cpp @@ -248,6 +248,7 @@ Util::config_advanced() bool Util::setGlobalConfig(INIParser& parser, string rootname, string keyname) { string value = parser.GetValue(rootname, keyname); + cout<<"the root name："<::iterator iter = Util::global_config.find(keyname); if (iter != Util::global_config.end()) { - return iter->second; + string value=iter->second; + value=Util::replace_all(value,"\"",""); + return value; } return ""; } @@ -269,7 +272,18 @@ bool Util::configure_new() ini_parser.ReadINI("conf.ini"); /*string value=ini_parser.GetValue("ghttp", "max_out_limit"); Util::global_config["max_out_limit"] = value;*/ + + Util::setGlobalConfig(ini_parser, "ghttp", "thread_num"); + + cout << "the current settings are as below: " << endl; + cout << "key : value" << endl; + cout << "------------------------------------------------------------" << endl; + for (map::iterator it = Util::global_config.begin(); it != Util::global_config.end(); ++it) + { + cout << it->first << " : " << it->second << endl; + } + cout << endl; Util::setGlobalConfig(ini_parser, "ghttp", "max_database_num"); Util::setGlobalConfig(ini_parser, "ghttp", "max_user_num"); Util::setGlobalConfig(ini_parser, "ghttp", "root_username"); @@ -281,9 +295,12 @@ bool Util::configure_new() Util::setGlobalConfig(ini_parser, "ghttp", "max_output_size"); Util::setGlobalConfig(ini_parser, "ghttp", "db_path"); Util::setGlobalConfig(ini_parser, "ghttp", "backup_path"); - Util::setGlobalConfig(ini_parser, "ghttp", "ip"); + + + // Util::setGlobalConfig(ini_parser, "ghttp", "ip"); Util::setGlobalConfig(ini_parser, "ghttp", "ip_allow_path"); Util::setGlobalConfig(ini_parser, "ghttp", "ip_deny_path"); + Util::setGlobalConfig(ini_parser, "ghttp", "ip_access_log"); Util::setGlobalConfig(ini_parser, "system", "version"); Util::setGlobalConfig(ini_parser, "system", "licensetype"); cout << "the current settings are as below: " << endl; @@ -1726,6 +1743,17 @@ Util::getTimeString() { return string(time_str); } +string +Util::getTimeString3() { + static const int max = 40; // max length of time string + char time_str[max]; + time_t timep; + time(&timep); + strftime(time_str, max, "%Y-%m-%d %H:%M:%S", localtime(&timep)); + return string(time_str); +} + + string Util::getTimeString2() { static const int max = 20; // max length of time string @@ -2507,100 +2535,160 @@ Util::update_transactionlog(std::string TID, std::string state, std::string end_ } string -Util::get_transactionlog() +Util::get_transactionlog(int page_no, int page_size) { + int totalSize = 0; + int totalPage = 0; + pthread_rwlock_rdlock(&transactionlog_lock); ifstream in; in.open(TRANSACTION_LOG_PATH, ios::in); + string line; + int startLine; + int endLine; + if(page_no < 1) + { + page_no = 1; + } + if(page_size < 1) + { + page_size = 10; + } + startLine = (page_no - 1)*page_size + 1; + endLine = page_no*page_size + 1; + //count total + while (getline(in, line, '\n')) + { + totalSize++; + } + in.close(); Document all; + Document::AllocatorType &allocator = all.GetAllocator(); all.SetObject(); Value a(kObjectType); Value darray(kArrayType); - string line; - Document d; - bool success = true; - while (getline(in, line)) { - cout << line << endl; - StringStream is(line.c_str()); - d.ParseStream(is); - Value rec(kObjectType); - if (d.HasMember("db_name")) - { - rec.AddMember("db_name", d["db_name"], all.GetAllocator()); - } - else - { - success = false; - darray.SetArray(); - break; - } - if (d.HasMember("TID")) - { - rec.AddMember("TID", d["TID"], all.GetAllocator()); - } - else - { - success = false; - darray.SetArray(); - break; - } - if (d.HasMember("user")) - { - rec.AddMember("user", d["user"], all.GetAllocator()); - } - else - { - success = false; - darray.SetArray(); - break; - } - if (d.HasMember("begin_time")) + if (totalSize > 0) + { + totalPage = (totalSize/page_size) + (totalSize%page_size == 0 ? 0 : 1); + if (page_no > totalPage) { - rec.AddMember("begin_time", d["begin_time"], all.GetAllocator()); + pthread_rwlock_unlock(&transactionlog_lock); + throw runtime_error("page_no more then max_page_no " + to_string(totalPage)); } - else + startLine = totalSize - page_size*page_no + 1; + endLine = totalSize - page_size*(page_no - 1) + 1; + if (startLine < 1) { - success = false; - darray.SetArray(); - break; - } - if (d.HasMember("state")) + startLine = 1; + } + // seek to start line; + in.open(TRANSACTION_LOG_PATH, ios::in); + int i_temp; + char buf_temp[1024]; + in.seekg(0, ios::beg); + for (i_temp = 1; i_temp < startLine; i_temp++) { - rec.AddMember("state", d["state"], all.GetAllocator()); + in.getline(buf_temp, sizeof(buf_temp)); } - else - { - success = false; - darray.SetArray(); - break; + + string line; + Document d; + bool success = true; + while (startLine < endLine && getline(in, line, '\n')) { + StringStream is(line.c_str()); + d.ParseStream(is); + Value rec(kObjectType); + if (d.HasMember("db_name")) + { + rec.AddMember("db_name", d["db_name"], allocator); + } + else + { + success = false; + darray.SetArray(); + break; + } + if (d.HasMember("TID")) + { + rec.AddMember("TID", d["TID"], allocator); + } + else + { + success = false; + darray.SetArray(); + break; + } + if (d.HasMember("user")) + { + rec.AddMember("user", d["user"], allocator); + } + else + { + success = false; + darray.SetArray(); + break; + } + if (d.HasMember("begin_time")) + { + rec.AddMember("begin_time", d["begin_time"], allocator); + } + else + { + success = false; + darray.SetArray(); + break; + } + if (d.HasMember("state")) + { + rec.AddMember("state", d["state"], allocator); + } + else + { + success = false; + darray.SetArray(); + break; + } + if (d.HasMember("end_time")) + { + rec.AddMember("end_time", d["end_time"], allocator); + } + else + { + success = false; + darray.SetArray(); + break; + } + darray.PushBack(rec, allocator); + startLine++; } - if (d.HasMember("end_time")) + in.close(); + if (success) { - rec.AddMember("end_time", d["end_time"], all.GetAllocator()); + all.AddMember("StatusCode", 0, allocator); + all.AddMember("StatusMsg", "Get Transaction log success", allocator); + all.AddMember("list", darray, allocator); + all.AddMember("totalSize", totalSize, allocator); + all.AddMember("totalPage", totalPage, allocator); + all.AddMember("pageNo", page_no, allocator); + all.AddMember("pageSize", page_size, allocator); } else { - success = false; - darray.SetArray(); - break; + all.AddMember("StatusCode", 1005, allocator); + all.AddMember("message", "error! Transaction log corrupted", allocator); } - darray.PushBack(rec, all.GetAllocator()); } - in.close(); - if (success) - { - all.AddMember("StatusCode", 0, all.GetAllocator()); - all.AddMember("StatusMsg", "Get Transaction log success", all.GetAllocator()); - all.AddMember("list", darray, all.GetAllocator()); - } - else { - all.AddMember("StatusCode", 1005, all.GetAllocator()); - all.AddMember("message", "error! Transaction log corrupted", all.GetAllocator()); - + darray.SetArray(); + all.AddMember("StatusCode", 0, allocator); + all.AddMember("StatusMsg", "Get Transaction log success", allocator); + all.AddMember("list", darray, allocator); + all.AddMember("totalSize", totalSize, allocator); + all.AddMember("totalPage", totalPage, allocator); + all.AddMember("pageNo", page_no, allocator); + all.AddMember("pageSize", page_size, allocator); } - StringBuffer buffer; Writer writer(buffer); all.Accept(writer); diff --git a/Util/Util.h b/Util/Util.h index 7c57fc9d..92a1be4b 100644 --- a/Util/Util.h +++ b/Util/Util.h @@ -377,6 +377,7 @@ public: static std::string getTimeName(); static std::string getTimeString(); static std::string getTimeString2(); + static std::string getTimeString3(); static int getRandNum(); static std::string get_folder_name(const std::string path, const std::string db_name); static std::string get_backup_time(const std::string path, const std::string db_name); @@ -489,10 +490,10 @@ public: static bool has_record_backuplog(std::string db_name); static void init_transactionlog(); - static int add_transactionlog(std::string db_name, std::string user, std::string TID, std::string begin_time, std::string status = "RUNNING", std::string end_time = "inf"); + static int add_transactionlog(std::string db_name, std::string user, std::string TID, std::string begin_time, std::string status = "RUNNING", std::string end_time = "INF"); static int delete_transactionlog(std::string db_name, std::string TID); static int update_transactionlog(std::string db_name, std::string status, std::string end_time); - static std::string get_transactionlog(); + static std::string get_transactionlog(int page_no, int page_size); static void abort_transactionlog(long end_time); static long int get_timestamp(std::string& line); diff --git a/backup.json b/backup.json index d8576026..b9df36c9 100644 --- a/backup.json +++ b/backup.json @@ -4,3 +4,4 @@ {"db_name": "csr","backup_interval": "600","last_backup_time": "2021-11-12 17:12:30","is_backup": "false"} {"db_name": "zqkg","backup_interval": "600","last_backup_time": "2021-11-12 17:12:30","is_backup": "false"} {"db_name": "yjzhkg","backup_interval": "600","last_backup_time": "2021-11-16 01:01:07","is_backup": "false"} +{"db_name": "rskg","backup_interval": "600","last_backup_time": "2021-12-02 16:37:19","is_backup": "false"} diff --git a/conf.ini b/conf.ini index 169f96cd..eb55bf3c 100644 --- a/conf.ini +++ b/conf.ini @@ -12,14 +12,14 @@ system_username="system" max_output_size=10000000 db_path="." backup_path="./backups" -ip_allow_path= -ip_deny_path= +ip_allow_path="" +ip_deny_path="ipDeny.config" +ip_access_log="logs/iplog/" #save the query log 0:not save 1:save the log save_log=0 - - [system] version=0.9.1 +licensetype=test [gserver] default_port=9000 [backup] @@ -27,7 +27,4 @@ default_port=9000 auto_backup=1 # max backups file, if the number of backup files more than the value, system will remove the old backup files. max_backups=4 -#backup interval,the unit is houre - - - +#backup interval,the unit is houre \ No newline at end of file diff --git a/docs/API.md b/docs/API.md index aed04141..d10a767b 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,2654 +1,945 @@ -## API Usage +**This Chapter guides you to use our API for accessing gStore. We provide HTTP api(suggested) and socket api, corresponding to ghttp and gserver respectively.** -​ +# HTTP API -### API instruction +Compired with socket API, HTTP API is more stable and more standard, and can maintain connection. Socket API can not guaratee correct transmission, so the network transmission is faster. ---- - -gStore provides API services to users through http and Socket services, and its components are ghttp and gServer. - -#### HTTP API instruction - -We now provide c++, Java, python, PHP, and nodejs apis for GHTTP. Please refer to the examples in `API/HTTP/CPP`, `API/HTTP/Java`, `API/HTTP /python`, `API/HTTP/PHP` and `API/HTTP /nodejs`. To use these examples, make sure you have generated an executable file. ** Next, use the `bin/ ghttp` command to start the ghttp service. ** If you know of a running available GHTTP server and try to connect to it, that's ok too. Then, for c++ and Java code, you need to compile the sample code in the directory `API/HTTP/CPP /example` and `API/HTTP/Java /example`. - -**For details on how to start and shut down ghttp, see [development documentation] - [Quick Start] - [http API service]** - -**After the API is started, the ghttp access address is as follow:** - -``` -http://serverip:port/ -``` - -`ServerIP` is the IP address of the gStore server, and `port` is the port on which ghttp is started. +## Easy Examples -#### Socker API instruction +We provide C++, java, python, php and nodejs API for ghttp now. Please refer to example codes in `api/http/cpp`, `api/http/java`, `api/http/python`, `api/http/php` and `api/http/nodejs`. To use these examples, please make sure that executables have already been generated. -We now provide c++, Java (python, PHP, nodejs) apis for gserver. See API /socket/ CPP, API /socket/ Java for example code. To use these examples, make sure you have generated an executable file. **Next, use the `bin/gserver -s` command to start the gserver service.**If you know of a running available gServer and try to connect to it, this is also ok. Then, for c++ and Java code, you need to compile the sample code in the directory `API/HTTP/CPP /example` and `API/HTTP/Java /example`. +Next, **start up ghttp service by using \texttt{bin/ghttp} command.** It is ok if you know a running usable ghttp server and try to connect to it. (you do not need to change anything if using examples, just by default). Then, for C++ and java code, you need to compile the example codes in the directory `api/http/cpp/example` and `api/http/java/example`. -**Start and close gServer** +Finally, go to the example directory and run the corresponding executables. All these four executables will connect to a specified ghttp server and do some load or query operations. Be sure that you see the query results in the terminal where you run the examples, otherwise please go to [Frequently Asked Questions](FAQ.md) for help or report it to us.(the report approach is described in [README](../README.md)) -**After the Socket API is started, you can connect through the Socket. The default port of gServer is 9000** +You are advised to read the example code carefully, as well as the corresponding Makefile. This will help you to understand the API, specially if you want to write your own programs based on the API interface. -
- -### HTTP API framework +- - - ---- +## API structure -The gStore HTTP API is placed in the API/HTTP directory of the gStore root directory and contains the following contents: +The HTTP API of gStore is placed in api/http directory in the root directory of gStore, whose contents are listed below: - gStore/api/http/ + + - cpp/ (the C++ API) + + - client.cpp (source code of C++ API) - - cpp/ (the C++ API) - - - client.cpp (C++ API's source code) - - - client.h - - - example/ (Example program using C++ API) - - GET-example.cpp + - client.h - - Benchmark.cpp + - example/ (small example program to show the basic idea of using the C++ API) + - GET-example.cpp - - POST-example.cpp + - Benchmark.cpp - - Makefile + - POST-example.cpp - - Makefile (Compile and build the lib) + - Makefile - - java/ (the Java API) + - Makefile (compile and build lib) + + - java/ (the Java API) - - client.java + - client.java - - lib/ + - lib/ - - src/ + - src/ - - Makefile + - Makefile - - jgsc/ + - jgsc/ - - GstoreConnector.java (Java API's source code) + - GstoreConnector.java - - example/ (A sample program using the Java API) + - example/ - - Benckmark.java + - Benckmark.java - GETexample.java - - POSTexample.java + - POSTexample.java - - Makefile + - Makefile - python/ (the Python API) - - example/ (A sample program using the Python API) + - example/ - - Benchmark.py + - Benchmark.py - - GET-example.py + - GET-example.py - - POST-example.py + - POST-example.py - - src/ + - src/ - - GstoreConnector.py (A sample program using the Python API) + - GstoreConnector.py - nodejs/ (the Nodejs API) - - GstoreConnector.js (Nodejs API's source code) - - - LICENSE + - GstoreConnector.js (source code of Nodejs API) - - package.json + - LICENSE - - README.md + - package.json - - example/ (A sample program using the Nodejs API) + - README.md - - POST-example.js + - example/ (small example to show the basic idea of using the Nodejs API) - - GET-example.js + - POST-example.js + - GET-example.js + - php/ (the Php API) - - example/ (A sample program using the php API) - - - Benchmark.php - - - POST-example.php + - example/ - - GET-example.php + - Benchmark.php - - src/ + - POST-example.php - - GstoreConnector.php (php API's source code) + - GET-example.php + - src/ + + - GstoreConnector.php - - - -
+## C++ API -### ghttp API instruction +#### Interface ---- - -#### API Interconnection Mode +To use the C++ API, please place the phrase `#include "client.h"` in your cpp code. Functions in client.h should be called like below: -> The ghttp interface adopts the `HTTP` protocol and supports multiple ways to access the interface. If the ghttp is started on the port `9000`, the interface interconnection content is as follows -> -> API address： -> -> ```json -> http://ip:9000/ -> ``` -> -> The interface supports both `GET` and `POST` requests, where `GET` requests place parameters in the URL and `POST` requests place parameters in the `body` request. -> -> - -> **Note: `GET` request parameters contain special characters, such as? , @,& and other characters, you need to use urlencode encoding, especially the `SPARQL` parameter must be encoded** -> -> - -#### API list - -| API name | Definition | Note | -| -------------------------- | ----------------------------------------- | ------------------------------------------------------------ | -| build | build graph database | The database file must be locally stored on the server | -| load | load graph database | Load the database into memory | -| unload | unload graph database | Unload the database from memory | -| monitor | monitor graph database | Count information about the specified database (such as the number of triples, etc.) | -| drop | drop graph database | Logical deletion and physical deletion can be performed | -| show | display graph database | Display a list of all databases | -| usermanage（added） | user management | Add, delete, or modify user information | -| showuser | display all user list | Display a list of all users | -| userprivilegemanage(added) | user privilege management | Add, delete, or modify user's privilege information | -| backup | backup database | backup database information | -| restore | restore database | restore database information | -| query | query database | Including query, delete, and insert | -| export | export database | Export database as NT file | -| login | login to database | It is used to authenticate user names and passwords | -| check（rewrite） | Detect ghttp heartbeat signal | | -| init（abandon） | Initialize system | This operation should not be initialized by ghttp | -| refresh（abandon） | reload database | This can be done by unload+ Load | -| parameter（abandon） | Set parameters for backup | All parameters are currently boiled down to the config.ini file and cannot be changed remotely | -| begin | Start transaction | Transaction starts and needs to be used in conjunction with TQuery | -| tquery | Querying the database (with transactions) | Data queries with transaction mode (insert and DELETE only) | -| commit | commit transactions | Commit the transaction after it completes | -| rollback | rollback transaction | Roll back the transaction to begin state | -| txnlog | Obtain transaction log information | Return transcation log information as json | -| checkpoint | write data to a disk | After an INSERT or delete operation is performed on the database, manually checkpoint is required | -| testConnect | testing connectivity | Used to check whether GHTTP is connected | -| getCoreVersion | get gStore version | Get the gStore version number | -| batchInsert | batch insert data | Batch insert NT data | -| batchRemove | batch delete data | Batch delete NT data | -| querylog（added） | Get query log information | Query log information is returned as json | -| ipmanage（added） | black/white list management | Maintains a blacklist and whitelist of IP addresses that access gStore | - -
- -#### API specific intruction - -> The input and output parameters of each interface are specified in this section. Assume that the IP address of the GHTTP server is 127.0.0.1 and the port is 9000 - -##### build -build datbase - -###### Brief description - -- Create a database based on existing NT files -- Files must exist on the gStore server - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode - -- GET/POST - -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON `structure - -###### Parameter - -| parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is **build** | -| db_name | yes | string | Database name (.db is not required) | -| db_path | yes | string | Database file path (can be an absolute path or a relative path. The relative path uses the gStore installation root directory as a reference directory) | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | - -###### Return value - -| Parameter name | Tpye | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "Import RDF file to database done." -} ``` +// initialize +GstoreConnector gc("127.0.0.1", 9000, "root", "123456"); -
- -##### check +// build a new database by a RDF file. +// note that the relative path is related to the server +gc.build("lubm", "data/lubm/lubm.nt"); -###### Brief description +// load the database that you built. +gc.load("lubm"); -- Check whether the GHTTP service is online +// to add, delete a user or modify the privilege of a user, operation must be done by the root user +gc.user("add_user", "user1", "111111"); -###### Request URL +// show all users +gc.showUser(); -- ` http://127.0.0.1:9000/ ` +// query +std::string res = gc.query("lubm", "json", sparql); +std::cout << res << std::endl; +// query and save the result in a file called "ans.txt" +gc.fquery("lubm", "json", sparql, "ans.txt"); -###### Request mode +// save the database if you have changed the database +gc.checkpoint("lubm"); -- ###### GET/POST +// show information of the database +gc.monitor("lubm"); -###### Parameter transfer mode +// show all databases +gc.show(); -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +// unload this database. +gc.unload("lubm"); -###### Parameter +// also, you can load some exist database directly and then query. +gc.load("lubm"); +res = gc.query("lubm", "json", sparql); +std::cout << res << std::endl; +gc.unload("lubm"); -| Parameter name | Mandatory | type | Note | -| :------------- | :-------- | :----- | ---------------------------------------- | -| operation | yes | string | Operation name, fixed value is **check** | +// drop the database directly +gc.drop("lubm", false); -###### Return value +// drop the database and leave a backup +gc.drop("lubm", true); -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | +// get CoreVersion and APIVersion +gc.getCoreVersion(); +gc.getAPIVersion(); - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "the ghttp server is running..." -} ``` +The original declaration of these functions are as below: -
- -##### load - -###### Brief description - -- Loading a database into memory is a prerequisite for many operations, such as Query, monitor, and so on. - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode - -- ###### GET/POST - -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | -------------------------------------- | -| operation | yes | string | Operation name, fixed value is**load** | -| db_name | yes | string | Database name (.db is not required) | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "Database loaded successfully." -} ``` +If request_type is "GET", the last parameter can be omitted: -
- -##### monitor - -###### Brief description - -- et database statistics (load database first) +GstoreConnector(std::string serverIP, int serverPort, std::string username, std::string password); -###### Request URL +std::string build(std::string db_name, std::string rdf_file_path, std::string request_type); -- ` http://127.0.0.1:9000/ ` +std::string load(std::string db_name, std::string request_type); +std::string unload(std::string db_name, std::string request_type); -###### Request mode +std::string user(std::string type, std::string username2, std::string addition, std::string request_type); -- GET/POST +std::string showUser(std::string request_type); -###### Parameter transfer mode +std::string query(std::string db_name, std::string format, std::string sparql, std::string request_type); -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +std::string drop(std::string db_name, bool is_backup, std::string request_type); -###### Parameter +std::string monitor(std::string db_name, std::string request_type); -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ----------------------------------------- | -| operation | yes | string | Operation name, fixed value is**monitor** | -| db_name | yes | string | Database name (.db is not required) | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | +std::string checkpoint(std::string db_name, std::string request_type); -###### Return value +std::string show(std::string request_type); -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| database | string | database name | -| creator | string | creator | -| built_time | string | create time | -| triple num | string | number of triples | -| entity num | int | number of entities | -| literal num | int | number of characters (attribute value) | -| subject num | int | number of subjects | -| predicate num | int | number of objects | -| connection num | int | number of connections | +std::string getCoreVersion(std::string request_type); +std::string getAPIVersion(std::string request_type); -###### Return sample +void fquery(std::string db_name, std::string format, std::string sparql, std::string filename, std::string request_type); -``` json -{ - "StatusCode": 0, - "StatusMsg": "success", - "database": "test_lubm", - "creator": "root", - "built_time": "2021-08-27 21:29:46", - "triple num": "99550", - "entity num": 28413, - "literal num": 0, - "subject num": 14569, - "predicate num": 17, - "connection num": 0 -} ``` -
- -##### unload - -###### Brief description - -- Unload the database from memory (all changes are flushed back to hard disk) - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode - -- GET/POST - -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ---------------------------------------- | -| operation | yes | string | Operation name, fixed value is**unload** | -| db_name | yes | string | Database name (.db is not required) | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | - -###### Return value +- - - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | +## Java API +#### Interface -###### Return sample +To use the Java API, please see gStore/api/http/java/src/jgsc/GstoreConnector.java. Functions should be called like below: -``` json -{ - "StatusCode": 0, - "StatusMsg": "Database unloaded." -} ``` +// initialize +GstoreConnector gc = new GstoreConnector("127.0.0.1", 9000, "root", "123456"); -
+// build a new database by a RDF file. +// note that the relative path is related to the server +gc.build("lubm", "data/lubm/lubm.nt"); -##### drop +// load the database that you built. +gc.load("lubm"); -###### Brief description +// to add, delete a user or modify the privilege of a user, operation must be done by the root user +gc.user("add_user", "user1", "111111"); -- Delete the database (either logically or physically) +// show all users +gc.showUser(); -###### Request URL +// query +String res = gc.query("lubm", "json", sparql); +System.out.println(res); -- ` http://127.0.0.1:9000/ ` +// query and save the result in a file called "ans.txt" +gc.fquery("lubm", "json", sparql, "ans.txt"); +// save the database if you have changed the database +gc.checkpoint("lubm"); -###### Request mode +// show information of the database +gc.monitor("lubm"); -- GET/POST +// show all databases +gc.show(); -###### Parameter transfer mode +// unload this database. +gc.unload("lubm"); -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +// also, you can load some exist database directly and then query. +gc.load("lubm"); +res = gc.query("lubm", "json", sparql); +System.out.println(res); +gc.unload("lubm"); -###### Parameter +// drop the database directly +gc.drop("lubm", false); -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**drop** | -| db_name | yes | string | Database name (.db is not required) | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | -| is_backup | no | string | True: Logical deletion, false: represents physical deletion (default true)，if it's logical deletion, change the file folder to .bak file foder, user can change the folder name to. Db and add the folder to the system database by calling bin/ ginit-db database name | +// drop the database and leave a backup +gc.drop("lubm", true); -###### Return value +// get CoreVersion and APIVersion +gc.getCoreVersion(); +gc.getAPIVersion(); -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "Database test_lubm dropped." -} ``` -
- -##### show - -###### Brief description - -- Display all database list - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode - -- GET/POST - -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | -------------------------------------- | -| operation | yes | string | Operation name, fixed value is**show** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | -###### Return value +The original declaration of these functions are as below: -| Parameter name | Type | Note | -| :------------------ | :-------- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| ResponseBody | JSONArray | JSON arrays (each of which is a database information) | -| -------- database | string | database name | -| ---------creator | string | creator | -| ---------built_time | string | create time | -| ---------status | string | database status | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "Get the database list successfully!", - "ResponseBody": [ - { - "database": "lubm", - "creator": "root", - "built_time": "2021-08-22 11:08:57", - "status": "loaded" - }, - { - "database": "movie", - "creator": "root", - "built_time": "2021-08-27 20:56:56", - "status": "unloaded" - } - ] -} ``` +If request_type is "GET", the last parameter can be omitted: -
+public class GstoreConnector(String serverIP, int serverPort, String username, String password); -##### usermanage（added） +public String build(String db_name, String rdf_file_path, String request_type); -###### Brief description +public String load(String db_name, String request_type); -- Manage users (including adding, deleting, and changing users) +public String unload(String db_name, String request_type); -###### Request URL +public String user(String type, String username2, String addition, String request_type); -- ` http://127.0.0.1:9000/ ` +public String showUser(String request_type); +public String query(String db_name, String format, String sparql, String request_type); -###### Request mode +public void fquery(String db_name, String format, String sparql, String filename, String request_type); -- GET/POST +public String drop(String db_name, boolean is_backup, String request_type); -###### Parameter transfer mode +public String monitor(String db_name, String request_type); -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +public String checkpoint(String db_name, String request_type); -###### Parameter +public String show(String request_type); -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**usermanage** | -| type | yes | string | Operation Type（1：adduser ，2：deleteUser 3：alterUserPassword） | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | -| op_username | yes | string | User name for the operation | -| op_password | yes | string | Password of the operation (if the password is to be changed, the password is the password to be changed) (If the operation contains special characters and the get request is adopted, the value must be urlencode-encoded) | +public String getCoreVersion(String request_type); -###### Return value +public String getAPIVersion(String request_type); -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ "StatusCode": 1004, "StatusMsg": "username already existed, add user failed."} ``` -
- -##### showuser - -###### Brief description - -- Display all user information - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode - -- GET/POST - -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**showuser** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | - -###### Return value +- - - -| Parameter name | Type | Note | -| :-------------------- | :-------- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| ResponseBody | JsonArray | JSON object array | -| ------username | string | user name | -| ------password | string | password | -| -----query_privilege | string | Query permissions (database names separated by commas) | -| ----update_privilege | string | Update permissions (database names separated by commas) | -| ----load_privilege | string | Load permissions (database names separated by commas) | -| ---unload_privilege | string | Unload permissions (database names separated by commas) | -| ----backup_privilege | string | Back up permissions (database names separated by commas) | -| ----restore_privilege | string | Restore permissions (database names separated by commas) | -| ---export_privilege | string | Export permissions (database names separated by commas) | +## Python API +#### Interface -###### Return sample +To use Python API, please see gStore/api/http/python/src/GstoreConnector.py. Functions should be called like following: -``` json -{ "StatusCode": 0, "StatusMsg": "success", "ResponseBody": [ { "username": "liwenjie", "password": "shuaige1982", "query privilege": "", "update privilege": "", "load privilege": "", "unload privilege": "", "backup privilege": "", "restore privilege": "", "export privilege": "" }, { "username": "liwenjie2", "password": "shuaige19888@&", "query privilege": "lubm,movie,", "update privilege": "lubm,movie,", "load privilege": "lubm,movie,", "unload privilege": "lubm,movie,", "backup privilege": "lubm,movie,", "restore privilege": "", "export privilege": "" }, { "username": "root", "password": "123456", "query privilege": "all", "update privilege": "all", "load privilege": "all", "unload privilege": "all", "backup privilege": "all", "restore privilege": "all", "export privilege": "all" } ]} ``` +# start a gc with given IP, Port, username and password +gc = GstoreConnector.GstoreConnector("127.0.0.1", 9000, "root", "123456") -
- -##### userprivilegemanage（added） - -###### Brief description - -- Manage user permissions (including adding, deleting, and changing users) +# build a database with a RDF graph +res = gc.build("lubm", "data/lubm/lubm.nt") -###### Request URL +# load the database +res = gc.load("lubm") -- ` http://127.0.0.1:9000/ ` +# to add, delete a user or modify the privilege of a user, operation must be done by the root user +res = gc.user("add_user", "user1", "111111") +# show all users +res = gc.showUser() -###### Request mode +# query +res = gc.query("lubm", "json", sparql) +print(res) -- GET/POST +# query and save the result in a file called "ans.txt" +gc.fquery("lubm", "json", sparql, "ans.txt") -###### Parameter transfer mode +# save the database if you have changed the database +res = gc.checkpoint("lubm") -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +# show information of the database +res = gc.monitor("lubm") -###### Parameter +# show all databases +res = gc.show() -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**userprivilegemanage** | -| type | yes | string | Operation type（1：add privilege，2：delete privilege 3：clear Privilege ） | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | -| op_username | yes | string | User name for the operation | -| privileges | no | string | Permissions to operate on (multiple permissions separated by commas) (can be null for clear Privilege)1:query,2:load,3:unload,4:update,5:backup,6:restore,7:export, you can set multi privileges by using , to split. | -| db_name | no | string | The database to operate on (this can be empty if it is clearPrivilege | +# unload the database +res = gc.unload("lubm") -###### Return value +# drop the database directly +res = gc.drop("lubm", False) -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | +# drop the database and leave a backup +res = gc.drop("lubm", True) +# get CoreVersion and APIVersion +res = gc.getCoreVersion(); +res = gc.getAPIVersion(); -###### Return sample - -``` json -{ "StatusCode": 0, "StatusMsg": "add privilege query successfully. \r\nadd privilege load successfully. \r\nadd privilege unload successfully. \r\nadd privilege update successfully. \r\nadd privilege backup successfully. \r\n"} ``` +The original declaration of these functions are as below: -
- -##### backup - -###### Brief description - -- Back up database - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode - -- GET/POST - -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**backup** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | database that need operaions | -| backup_path | no | string | The backup file path can be relative or absolute. The relative path uses the gStore root directory as reference. The default path is the backup directory in the gStore root directory | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| backupfilepath | string | Backup file path (this value can be used as the input parameter value for restore) | - - -###### Return sample - -``` json -{ "StatusCode": 0, "StatusMsg": "Database backup successfully.", "backupfilepath": "testbackup/lubm.db_210828211529"} ``` +If request_type is "GET", the last parameter can be omitted: -
- -##### restore - -###### Brief description +public class GstoreConnector(self, serverIP, serverPort, username, password): -- Restore database +def build(self, db_name, rdf_file_path, request_type): -###### Request URL +def load(self, db_name, request_type): -- ` http://127.0.0.1:9000/ ` +def unload(self, db_name, request_type): +def user(self, type, username2, addition, request_type): -###### Request mode +def showUser(self, request_type): -- GET/POST +def query(self, db_name, format, sparql, request_type): -###### Parameter transfer mode +def fquery(self, db_name, format, sparql, filename, request_type): -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +def drop(self, db_name, is_backup, request_type): -###### Parameter +def monitor(self, db_name, request_type): -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**restore** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | database that need operations | -| backup_path | no | string | The full time-stamped path of the backup file (can be a relative path or an absolute path. The relative path is based on the gStore root directory). The default path is the backup directory in the gStore root directory | +def checkpoint(self, db_name, request_type): -###### Return value +def show(self, request_type): -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | +def getCoreVersion(self, request_type): +def getAPIVersion(self, request_type): -###### Return sample - -``` json -{ "StatusCode": 0, "StatusMsg": "Database restore successfully."} ``` -
- -##### query - -###### Brief description - -- query the database - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode - -- GET/POST - -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**query** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | database that need operations | -| format | no | string | The result set returns in json, HTML, and file format. The default is JSOn | -| sparql | yes | string | Sparql statement to execute (SPARQL requires URL encoding if it is a GET request) | +- - - -###### Return value +## Nodejs API -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| head | JSON | head information | -| results | JSON | Result information (see Return Sample for details) | +#### Interface +Before using Nodejs API, type `npm install request` and `npm install request-promise` under the nodejs folder to add the required module. -###### Return sample +To use Nodejs API, please see gStore/api/http/nodejs/GstoreConnector.js. Functions should be called like following: -``` json -{ "head": { "link": [], "vars": [ "x" ] }, "results": { "bindings": [ { "x": { "type": "uri", "value": "十面埋伏" } }, { "x": { "type": "uri", "value": "投名状" } }, { "x": { "type": "uri", "value": "如花" } } ] }, "StatusCode": 0, "StatusMsg": "success"} ``` +// start a gc with given IP, Port, username and password +gc = new GstoreConnector("127.0.0.1", 9000, "root", "123456"); -
- -##### export - -###### Brief description +// build a database with a RDF graph +res = gc.build("lubm", "data/lubm/lubm.nt"); -- export database +// load the database +res = gc.load("lubm"); -###### Request URL +// to add, delete a user or modify the privilege of a user, operation must be done by the root user +res = gc.user("add_user", "user1", "111111"); -- ` http://127.0.0.1:9000/ ` +// show all users +res = gc.showUser(); +// query +res = gc.query("lubm", "json", sparql); +console.log(JSON.stringify(res,",")); -###### Request mode +// save the database if you have changed the database +res = gc.checkpoint("lubm"); -- GET/POST +// show information of the database +res = gc.monitor("lubm"); -###### Parameter transfer mode +// show all databases +res = gc.show(); -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +// unload the database +res = gc.unload("lubm"); -###### Parameter +// drop the database directly +res = gc.drop("lubm", false); -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ----------------------------------------- | -| operation | yes | string | Operation name, fixed value is**restore** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | database that need operations | -| db_path | no | string | Export path (gstore root by default) | +// drop the database and leave a backup +res = gc.drop("lubm", true); -###### Return value +// get CoreVersion and APIVersion +res = gc.getCoreVersion(); +res = gc.getAPIVersion(); -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| filepath | string | Path for exporting files | - - -###### Return sample - -``` json -{ "StatusCode": 0, "StatusMsg": "Export the database successfully.", "filepath": "export/lubm_210828214603.nt"} ``` +The original declaration of these functions are as below: - - -
- -##### login - -###### Brief description - -- Login user (verify username and password) - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode - -- GET/POST - -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | --------------------------------------- | -| operation | yes | string | Operation name, fixed value is**login** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ "StatusCode": 1001, "StatusMsg": "wrong password."} ``` +If request_type is "GET", the last parameter can be omitted: -
- -##### begin - -###### Brief description +class GstoreConnector(ip = '', port, username = '', password = ''); -- start transaction +async build(db_name = '', rdf_file_path = '', request_type); -###### Request URL +async load(db_name = '', request_type); -- ` http://127.0.0.1:9000/ ` +async unload(db_name = '', request_type); +async user(type = '', username2 = '' , addition = '' , request_type); -###### Request mode +async showUser(request_type); -- GET/POST +async query(db_name = '', format = '' , sparql = '' , request_type); -###### Parameter transfer mode +async drop(db_name = '', is_backup , request_type); -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +async monitor(db_name = '', request_type); -###### Parameter +async checkpoint(db_name = '', request_type); -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**begin** | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | Database name | -| isolevel | yes | string | Transaction isolation level 1:RC(read committed) 2:SI(snapshot isolation) 3:SR(seriablizable） | +async show(request_type); -###### Return value +async getCoreVersion(request_type); -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| TID | string | Transaction ID(this ID is important enough to take as a parameter | +async getAPIVersion(request_type); - -###### Return sample - -``` json -{ "StatusCode": 1001, "StatusMsg": "wrong password."} ``` -
- -##### tquery - -###### Brief description - -- query the transaction type - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode - -- GET/POST - -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ---------------------------------------- | -| operation | yes | string | Operation name, fixed value is**tquery** | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | Database name | -| tid | yes | string | Transaction ID | -| sparql | yes | string | sparql statement | - -###### Return value +- - - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | +## Php API +#### Interface -###### Return sample +To use Php API, please see gStore/api/http/php/src/GstoreConnector.php. Functions should be called like following: -``` json -{ "result": "", "StatusCode": 0, "StatusMsg": "success"} ``` +// start a gc with given IP, Port, username and password +$gc = new GstoreConnector("127.0.0.1", 9000, "root", "123456"); -
- -##### commit - -###### Brief description - -- submit transaction - -###### Request URL +// build a database with a RDF graph +$res = $gc->build("lubm", "data/lubm/lubm.nt"); +echo $res . PHP_EOL; -- ` http://127.0.0.1:9000/ ` +// load rdf +$ret = $gc->load("test", $username, $password); +echo $ret . PHP_EOL; +// to add, delete a user or modify the privilege of a user, operation must be done by the root user +//$res = $gc->user("add_user", "user1", "111111"); +//echo $res . PHP_EOL; -###### Request mode +// show all users +$res = $gc->showUser(); +echo $res. PHP_EOL; -- GET/POST +// query +$res = $gc->query("lubm", "json", $sparql); +echo $res. PHP_EOL; -###### Parameter transfer mode +// query and save the result in a file called "ans.txt" +$gc->fquery("lubm", "json", $sparql, "ans.txt"); -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +// save the database if you have changed the database +$res = $gc->checkpoint("lubm"); +echo $res. PHP_EOL; -###### Parameter +// show information of the database +$res = $gc->monitor("lubm"); +echo $res. PHP_EOL; -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ---------------------------------------- | -| operation | yes | string | Operation name, fixed value is**commit** | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | Database name | -| tid | yes | string | Transaction ID | +// show all databases +$res = $gc->show(); +echo $res. PHP_EOL; -###### Return value +// unload the database +$res = $gc->unload("lubm"); +echo $res. PHP_EOL; -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | +// drop the database directly +$res = $gc->drop("lubm", false); +echo $res. PHP_EOL; +// drop the database and leave a backup +$res = $gc->drop("lubm", true); +echo $res. PHP_EOL; -###### Return sample +// get CoreVersion and APIVersion +$res = $gc->getCoreVersion(); +echo $res. PHP_EOL; +$res = $gc->getAPIVersion(); +echo $res. PHP_EOL; -``` json -{ "StatusCode": 0, "StatusMsg": "transaction commit success. TID: 1"} ``` -
- -##### rollback - -###### Brief description - -- Rollback trasnsaction - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode - -- GET/POST +The original declaration of these functions are as below: -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**rollback** | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | Database name | -| tid | yes | string | Transaction ID | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ "StatusCode": 0, "StatusMsg": "transaction rollback success. TID: 2"} ``` +If request_type is "GET", the last parameter can be omitted: -
+class GstoreConnector($ip, $port, $username, $password) -##### txnlog +function build($db_name, $rdf_file_path, $request_type) -###### Brief description +function load($db_name, $request_type) -- Get transaction logs (this function only works for root user) +function unload($db_name, $request_type) -###### Request URL +function user($type, $username2, $addition, $request_type) -- ` http://127.0.0.1:9000/ ` +function showUser($request_type) +function query($db_name, $format, $sparql, $request_type) -###### Request mode +function fquery($db_name, $format, $sparql, $filename, $request_type) -- GET/POST +function drop($db_name, $is_backup, $request_type) -###### Parameter transfer mode +function monitor($db_name, $request_type) -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +function checkpoint($db_name, $request_type) -###### Parameter +function show($request_type) -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ---------------------------------------- | -| operation | yes | string | Operation name, fixed value is**txnlog** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | +function getCoreVersion($request_type) -###### Return value +function getAPIVersion($request_type) -| Parameter name | Type | Note | -| :------------- | :-------- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| list | JSONArray | Log JSON array | - - -###### Return sample - -``` json -{ "StatusCode": 0, "StatusMsg": "Get Transaction log success", "list": [ { "db_name": "lubm2", "TID": "1", "user": "root", "begin_time": "1630376221590", "state": "COMMITED", "end_time": "1630376277349" }, { "db_name": "lubm2", "TID": "2", "user": "root", "begin_time": "1630376355226", "state": "ROLLBACK", "end_time": "1630376379508" } ]} ``` +--- +--- +# Socket API -
- -##### checkpoint - -###### Brief description - -- Received Flush data back to hard disk (to make data final) - -###### Request URL - -- ` http://127.0.0.1:9000/ ` - - -###### Request mode +**This APIis not maintained now.** -- GET/POST +## Easy Examples -###### Parameter transfer mode +We provide JAVA, C++, PHP and Python API for gStore now. Please refer to example codes in `api/socket/cpp/example`, `api/socket/java/example`, `api/socket/php` and `api/socket/python`. To use C++ and Java examples to have a try, please ensure that executables have already been generated. Otherwise, just type `make APIexample` in the root directory of gStore to compile the codes, as well as API. -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +Next, **start up a gStore server by using `./gserver` command.** It is ok if you know a running usable gStore server and try to connect to it, but notice that **the server ip and port of server and client must be matched.**(you do not need to change any thing if using examples, just by default) Then, you need to compile the example codes in the directory gStore/api/socket. We provide a utility to do this, and you just need to type `make APIexample` in the root directory of gStore. Or you can compile the codes by yourself, in this case please go to gStore/api/socket/cpp/example/ and gStore/socket/api/java/example/, respectively. -###### Parameter +Finally, go to the example directory and run the corresponding executables. For C++, just use `./example` command to run it. And for Java, use `make run` command or `java -cp ../lib/GstoreJavaAPI.jar:. JavaAPIExample` to run it. Both the two executables will connect to a specified gStore server and do some load or query operations. Be sure that you see the query results in the terminal where you run the examples, otherwise please go to [Frequently Asked Questions](FAQ.md) for help or report it to us.(the report approach is described in [README](../README.md)) -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | -------------------------------------------- | -| operation | yes | string | Operation name, fixed value is**checkpoint** | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | Database name | +You are advised to read the example code carefully, as well as the corresponding Makefile. This will help you to understand the API, specially if you want to write your own programs based on the API interface. -###### Return value +- - - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | +## API structure +The socket API of gStore is placed in api/socket directory in the root directory of gStore, whose contents are listed below: -###### Return sample +- gStore/api/socket/ + + - cpp/ (the C++ API) + + - src/ (source code of C++ API, used to build the lib/libgstoreconnector.a) -``` json -{ "StatusCode": 0, "StatusMsg": "Database saved successfully."} -``` + - GstoreConnector.cpp (interfaces to interact with gStore server) -
+ - GstoreConnector.h -##### testConnect + - Makefile (compile and build lib) + + - lib/ (where the static lib lies in) -###### Brief description + - .gitignore -- Test whether the server can connect (for workbench) + - libgstoreconnector.a (only exist after compiled, you need to link this lib when you use the C++ API) -###### Request URL + - example/ (small example program to show the basic idea of using the C++ API) -- ` http://127.0.0.1:9000/ ` + - CppAPIExample.cpp + - Makefile -###### Request mode + - java/ (the Java API) -- GET/POST + - src/ (source code of Java API, used to build the lib/GstoreJavaAPI.jar) + + - jgsc/ (the package which you need to import when you use the Java API) + + - GstoreConnector.java (interfaces to interact with gStore server) -###### Parameter transfer mode + - Makefile (compile and build lib) -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure + - lib/ -###### Parameter + - .gitignore -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | --------------------------------------------- | -| operation | yes | string | Operation name, fixed value is**testConnect** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | + - GstoreJavaAPI.jar (only exist after compiled, you need to include this JAR in your class path) -###### Return value + - example/ (small example program to show the basic idea of using the Java API) -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| CoreVersion | string | Core version number | -| licensetype | string | License type (Open Source or Enterprise) | + - JavaAPIExample.cpp + - Makefile -###### Return sample + - php/ (the PHP API) -``` json -{ "StatusCode": 0, "StatusMsg": "success", "CoreVersion": "0.9.1", "licensetype": "\"opensource\""} -``` + - PHPAPIExxample.php (small example program to show the basic idea of using the PHP API) -
+ - GstoreConnector.php (source code of PHP API) -##### getCoreVersion + - python/ (the python API) -###### Brief description + - src/ (source code of Python API) -- Get the server version number (for Workbench) + - GstoreConnector.py -###### Request URL + - lib/ -- ` http://127.0.0.1:9000/ ` + - example/ (small example program to show the basic idea of using the Python API) + - PythonAPIExample.py -###### Request mode -- GET/POST +- - - -###### Parameter transfer mode +## C++ API -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +#### Interface -###### Parameter +To use the C++ API, please place the phrase `#include "GstoreConnector.h"` in your cpp code. Functions in GstoreConnector.h should be called like below: -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**getCoreVersion** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | +``` +// initialize the Gstore server's IP address and port. +GstoreConnector gc("127.0.0.1", 3305); -###### Return value +// build a new database by a RDF file. +// note that the relative path is related to gserver. +gc.build("LUBM10.db", "example/LUBM_10.n3"); -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| CoreVersion | string | Core version number | +// then you can execute SPARQL query on this database. +std::string sparql = "select ?x where \ + { \ + ?x . \ + ?y . \ + ?x ?y. \ + ?z ?y. \ + ?z . \ + ?z ?w. \ + ?w . \ + }"; +std::string answer = gc.query(sparql); +// unload this database. +gc.unload("LUBM10.db"); -###### Return sample +// also, you can load some exist database directly and then query. +gc.load("LUBM10.db"); -``` json -{ "StatusCode": 0, "StatusMsg": "success", "CoreVersion": "0.9.1"} +// query a SPARQL in current database +answer = gc.query(sparql); ``` -
+The original declaration of these functions are as below: -##### batchInsert - -###### Brief description +``` +GstoreConnector(); +GstoreConnector(string _ip, unsigned short _port); +GstoreConnector(unsigned short _port); +bool load(string _db_name); +bool unload(string _db_name); +bool build(string _db_name, string _rdf_file_path); +string query(string _sparql); +``` -- batch insert data +Notice: -###### Request URL +1. When using GstoreConnector(), the default value for ip and port is 127.0.0.1 and 3305, respectively. -- ` http://127.0.0.1:9000/ ` +2. When using build(), the rdf_file_path(the second parameter) should be related to the position where gserver lies in. +3. Please remember to unload the database you have loaded, otherwise things may go wrong.(the errors may not be reported!) -###### Request mode +#### Compile -- GET/POST +You are advised to see gStore/api/socket/cpp/example/Makefile for instructions on how to compile your code with the C++ API. Generally, what you must do is compile your own code to object with header in the C++ API, and link the object with static lib in the C++ API. -###### Parameter transfer mode +Let us assume that your source code is placed in test.cpp, whose position is ${TEST}, while the gStore project position is ${GSTORE}/gStore.(if using devGstore as name instead of gStore, then the path is ${GSTORE}/devGstore) Please go to the ${TEST} directory first: -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +> Use `g++ -c -I${GSTORE}/gStore/api/socket/cpp/src/ test.cpp -o test.o` to compile your test.cpp into test.o, relative API header is placed in api/socket/cpp/src/. -###### Parameter +> Use `g++ -o test test.o -L${GSTORE}/gStore/api/socket/cpp/lib/ -lgstoreconnector` to link your test.o with the libgstoreconnector.a(a static lib) in api/socket/cpplib/. -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**batchInsert** | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | Database name | -| file | yes | string | The data NT file to insert (can be a relative or absolute path) | +Then you can type `./test` to execute your own program, which uses our C++ API. It is also advised for you to place relative compile commands in a Makefile, as well as other commands if you like. -###### Return value +- - - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| success_num | string | Number of successful executions | +## Java API +#### Interface -###### Return sample +To use the Java API, please place the phrase `import jgsc.GstoreConnector;` in your java code. Functions in GstoreConnector.java should be called like below: -``` json -{ "StatusCode": 0, "StatusMsg": "Batch Insert Data Successfully.", "success_num": "25"} ``` +// initialize IP address and port of the Gstore server. +GstoreConnector gc = new GstoreConnector("127.0.0.1", 3305); -
- -##### batchRemove - -###### Brief description +// build a new database by a RDF file. +// note that the relative path is related to gserver. +gc.build("LUBM10.db", "example/LUBM_10.n3"); -- batch remove data +// then you can execute SPARQL query on this database. +String sparql = "select ?x where " + + "{" + + "?x . " + + "?y . " + + "?x ?y. " + + "?z ?y. " + + "?z . " + + "?z ?w. " + + "?w . " + + "}"; +String answer = gc.query(sparql); -###### Request URL +// unload this database. +gc.unload("LUBM10.db"); -- ` http://127.0.0.1:9000/ ` +// also, you can load some exist database directly and then query. +gc.load("LUBM10.db"); - -###### Request mode - -- GET/POST - -###### Parameter transfer mode - -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**batchRemove** | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | -| db_name | yes | string | Database name | -| file | yes | string | Data NT files to be deleted (can be relative or absolute paths) | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| success_num | int | Number of successful executions | - - -###### Return sample - -``` json -{ "StatusCode": 0, "StatusMsg": "Batch Remove Data Successfully.", "success_num": "25"} +// query a SPARQL in current database +answer = gc.query(sparql); ``` -
- -##### shutdown +The original declaration of these functions are as below: -###### Brief description - -- close ghttp - -###### Request URL +``` +GstoreConnector(); +GstoreConnector(string _ip, unsigned short _port); +GstoreConnector(unsigned short _port); +bool load(string _db_name); +bool unload(string _db_name); +bool build(string _db_name, string _rdf_file_path); +string query(string _sparql); +``` -- ` http://127.0.0.1:9000/shutdown `[Note, address change] +Notice: +1. When using GstoreConnector(), the default value for ip and port is 127.0.0.1 and 3305, respectively. -###### Request mode +2. When using build(), the rdf_file_path(the second parameter) should be related to the position where gserver lies in. -- GET/POST +3. Please remember to unload the database you have loaded, otherwise things may go wrong.(the errors may not be reported!) -###### Parameter transfer mode +#### Compile -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +You are advised to see gStore/api/socket/java/example/Makefile for instructions on how to compile your code with the Java API. Generally, what you must do is compile your own code to object with jar file in the Java API. -###### Parameter +Let us assume that your source code is placed in test.java, whose position is ${TEST}, while the gStore project position is ${GSTORE}/gStore.(if using devGstore as name instead of gStore, then the path is ${GSTORE}/devGstore) Please go to the ${TEST} directory first: -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| username | yes | string | user name（default user name is system) | -| password | yes | string | Password（This password need to be viewed in the server's system.db/password[port].txt file，for example, if the port is 9000，then check password in password9000.txt file) | +> Use `javac -cp ${GSTORE}/gStore/api/socket/java/lib/GstoreJavaAPI.jar test.java` to compile your test.java into test.class with the GstoreJavaAPI.jar(a jar package used in Java) in api/java/lib/. -###### Return value +Then you can type `java -cp ${GSTORE}/gStore/api/socket/java/lib/GstoreJavaAPI.jar:. test` to execute your own program(notice that the ":." in command cannot be neglected), which uses our Java API. It is also advised for you to place relative compile commands in a Makefile, as well as other commands if you like. -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | +- - - -###### Return sample +## PHP API + +#### Interface + +To use the PHP API, please place the phrase `include('GstoreConnector,php');` in your php code. Functions in +GstoreConnector.php should be called like below: + +``` +// initialize IP address and port of the Gstore server. +$gc = new Connector("127.0.0.1", 3305); +// build a new database by a RDF file. +// note that the relative path is related to gserver. +$gc->build("LUBM10", "example/LUBM_10.n3"); +// then you can execute SPARQL query on this database. +$sparql = "select ?x where " + "{" + + "?x . " + + "?y . " + + "?x ?y. " + + "?z ?y. " + + "?z . " + + "?z ?w. " + + "?w . " + + "}"; +$answer = gc->query($sparql); +//unload this database. +$gc->unload("LUBM10"); +//also, you can load some exist database directly and then query. +$gc->load("LUBM10");// query a SPARQL in current database +$answer = gc->query(sparql); +``` + +The original declaration of these functions are as below: + +``` +class Connector { + public function __construct($host, $port); + public function send($data); + public function recv(); + public function build($db_name, $rdf_file_path); + public function load($db_name); + public function unload($db_name); + public function query($sparql); + public function __destruct(); +} -``` json -If no value is returned, no information is received by default on success, and an error JSON message is returned on failure ``` -
- -##### querylog - -###### Brief description - -- Obtaining query Logs - -###### Request URL - -`http://127.0.0.1:9000` - -###### Request mode +Notice: -- GET/POST +1. When using Connector(), the default value for ip and port is 127.0.0.1 and 3305, respectively. -###### Parameter transfer mode +2. When using build(), the rdf_file_path(the second parameter) should be related to the position where gserver lies in. -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure +3. Please remember to unload the database you have loaded, otherwise things may go wrong.(the errors may not be reported!) -###### Parameter +#### Run -| Parameter name | Mandatory | Type | Note | -| -------------- | --------- | ------ | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**querylog** | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | -| date | yes | string | Date format is yyyyMMdd | -| pageNo | yes | int | Page number. The value ranges from 1 to N. The default value is 1 | -| pageSize | yes | int | Number of pages. The value ranges from 1 to N. The default value is 10 | +You can see gStore/api/socket/php/PHPAPIExample for instructions on how to use PHP API. PHP script does not need compiling. You can run PHP file directly or use it in your web project. -###### Return value - -| Parameter | Type | Note | -| :------------ | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| totalSize | int | Total number | -| totalPage | int | Total page number | -| pageNo | int | Current page number | -| pageSize | int | Each page | -| list | Array | Log array | -| QueryDateTime | string | Query date/time | -| Sparql | string | SPARQL statement | -| Format | string | Query return format | -| RemoteIP | string | Request IP | -| FileName | string | Query result set files | -| QueryTime | int | Time (ms) | -| AnsNum | int | Result number | +- - - -###### Return sample +## Python API + +#### Interface + +To use the Python API, please place the phrase `from GstoreConnector import GstoreConnector` in your python code. Functions in GstoreConnector.py should be called like below: + +``` +// initialize IP address and port of the Gstore server. +gc = GstoreConnector('127.0.0.1', 3305) +// build a new database by a RDF file. +// note that the relative path is related to gserver. +gc.build('LUBM10', 'data/LUBM_10.n3') +// then you can execute SPARQL query on this database. +$sparql = "select ?x where " + "{" + + "?x . " + + "?y . " + + "?x ?y. " + + "?z ?y. " + + "?z . " + + + "?z ?w. " + + "?w . " + + "}"; +answer = gc.query(sparql) +//unload this database. +gc.unload('LUBM10') +//also, you can load some exist database directly and then query. +gc.load('LUBM10')// query a SPARQL in current database +answer = gc.query(sparql) +``` + +The original declaration of these functions are as below: + +``` +class GstoreConnector { + def _connect(self) + def _disconnect(self) + def _send(self, msg): + def _recv(self) + def _pack(self, msg): + def _communicate(f): + def __init__(self, ip='127.0.0.1', port=3305): + @_communicate + def test(self) + @_communicate + def load(self, db_name) + @_communicate + def unload(self, db_name) + @_communicate + def build(self, db_name, rdf_file_path) + @_communicate + def drop(self, db_name) + @_communicate + def stop(self) + @_communicate + def query(self, sparql) + @_communicate + def show(self, _type=False) +} -```json -{ "StatusCode":0, "StatusMsg":"Get query log success", "totalSize":64, "totalPage":13, "pageNo":2, "pageSize":5, "list":[ { "QueryDateTime":"2021-11-16 14:55:52:90ms:467microseconds", "Sparql":"select ?name where { ?name <不喜欢> . }", "Format":"json", "RemoteIP":"183.67.4.126", "FileName":"140163774674688_20211116145552_847890509.txt", "QueryTime":0, "AnsNum":2 } ...... ]} ``` -
- -##### ipmanage - -###### Brief description - -- Blacklist and whitelist management - -###### Request URL - -`http://127.0.0.1:9000` +Notice: -###### Request mode +1. When using GstoreConnector(), the default value for ip and port is 127.0.0.1 and 3305, respectively. -- GET/POST +2. When using build(), the rdf_file_path(the second parameter) should be related to the position where gserver lies in. -###### Parameter transfer mode +3. Please remember to unload the database you have loaded, otherwise things may go wrong.(the errors may not be reported!) -- GET request, the parameters are passed directly as the URL -- POST request, `raw` in `body` in `Httprequest`, passed as`JSON ` structure - -###### Parameter - -查询黑白名单 - -| Parameter name | Mandatory | Type | Note | -| -------------- | --------- | ------ | ------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**ipmanage** | -| username | yes | string | User name | -| password | yes | string | Password (plain text) | -| type | yes | string | Operation type, fixed value is**1** | - -保存黑白名单 - -| Parameter name | Mandatory | Type | Note | -| -------------- | --------- | ------ | ------------------------------------------------------------ | -| operation | yes | string | Operation name, fixed value is**ipmanage** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | -| type | yes | string | Operation type, fixed value is**2** | -| whiteIPs | yes | string | Whitelist (multiple with **,** split, support range configuration, use **-** connection such as: IP1-1P2) | -| blackIPs | yes | string | Blacklist (multiple with **,** split, support range configuration, use **-** connection such as: IP1-1P2) | - -```json -//保存POST示例{ "operation": "ipmanage", "username": "root", "password": "123456", "type": "2", "whiteIPs":"127.0.0.1,183.67.4.126-183.67.4.128", "blackIPs": "192.168.1.141"} -``` - - - -###### Return value - -| Parameter | Type | Note | -| :----------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| ResponseBody | Object | Return data (only for queries) | -| whiteIPs | array | Whitelist | -| blackIPs | array | Blacklist | - -###### Return sample - -```json -// 查询黑白名单返回{ "StatusCode": 0, "StatusMsg": "success", "ResponseBody": { "whiteIPs": [ "127.0.0.1", "183.67.4.126-183.67.4.128" ], "blackIPs": [ "192.168.1.141" ] }}// 保存黑白名单返回{ "StatusCode": 0, "StatusMsg": "success"} -``` - -
- -##### Attached table 1 return value code table - -| Code value | Definition | -| ---------- | ------------------------------------------- | -| 0 | Success | -| 1000 | The method type is not support | -| 1001 | Authentication Failed | -| 1002 | Check Privilege Failed | -| 1003 | Param is illegal | -| 1004 | The operation conditions are not satisfied | -| 1005 | Operation failed | -| 1006 | Add privilege Failed | -| 1007 | Loss of lock | -| 1008 | Transcation manage Failed | -| 1100 | The operation is not defined | -| 1101 | IP Blocked | -| | | - -
- -### C++ HTTP API - ---- - -To use the C++ API, put the phrase '#include "client.h" in your CPP code, as shown below： - -**Construct the initialization function** - - - GstoreConnector(std::string serverIP, int serverPort, std::string username, std::string password); - Function: Initialize - Parameter Definition：[Server IP], [GHTTP port on the server], [Username], [password] - Example：GstoreConnector gc("127.0.0.1", 9000, "root", "123456"); - -**Build database: build** - - - std::string build(std::string db_name, std::string rdf_file_path, std::string request_type); - Function：Create a new database from an RDF file - Parameter Definition：[database name], [.nt file path], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：gc.build("lubm", "data/lubm/lubm.nt"); - -**Load database: load** - - std::string load(std::string db_name, std::string request_type); - Function：Load the database you created - Parameter Definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：gc.load("lubm"); - -**Stopping database loading: unload** - - - std::string unload(std::string db_name, std::string request_type); - Function：Stopping database loading - Parameter Definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：gc.unload("lubm"); - -**User management: user** - - std::string user(std::string type, std::string username2, std::string addition, std::string request_type); - Function：The root user can add, delete, or modify the user's permission only - 1.Add or delete users： - Parameter Definition：["add_user" adds user, "delete_user" deletes user],[user name],[password],[request type "GET" and "POST ", if request type "GET" can be omitted] - Example：gc.user("add_user", "user1", "111111"); - 2.Modify user's privilege： - Parameter Definition：["Add_query" adds query permission, "delete_query" deletes query permission, "add_load" adds load permission, "delete_load" deletes load permission, "add_unload" adds no load permission, "delete_unload" deletes no load permission, "Add_update" adds update permission, "delete_update" deletes update permission, "add_backup" adds backup permission, "delete_bakup" deletes backup permission, "add_restore" adds restore permission, "Delete_restore" deletes restore permission,"add_export" adds export permission," delete_export" deletes export permission],[user name],[database name],[Request type "GET" and "POST ", if the request type is "GET", it can be omitted.] - Example：gc.user("add_query", "user1", "lubm"); - -**Display user: showUser** - - std::string showUser(std::string request_type); - Function：Display all Users - Parameter Definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：gc.showUser(); - -**Database query: query** - - std::string query(std::string db_name, std::string format, std::string sparql, std::string request_type); - Function：Query the database - Parameter Definition：[database name], [query result type JSON, HTML or text], [SPARQL statement], [Request type "GET" and "POST ", if the request type is "GET", it can be omitted] - Example： - std::string res = gc.query("lubm", "json", sparql); - std::cout << res << std::endl; //output result - -**Deleting a Database: drop** - - std::string drop(std::string db_name, bool is_backup, std::string request_type); - Function：Delete the database directly or delete the database while leaving a backup - Parameter Definition：[database name], [false not backup, true backup], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：gc.drop("lubm", false); //Delete the database without leaving a backup - -**Monitoring database: monitor** - - std::string monitor(std::string db_name, std::string request_type);Function：Displays information for a specific database. - Parameter Definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted]Example：gc.monitor("lubm"); - -**Save the database: checkpoint** - - std::string checkpoint(std::string db_name, std::string request_type);Function：If you change the database, save the databas Parameter Definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted]Example：gc.checkpoint("lubm"); - -**Show the database: show** - - std::string show(std::string request_type);Function：Displays all created databases Parameter Definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"] Example：gc.show(); - -**The kernel version information is displayed: getCoreVersion** - - std::string getCoreVersion(std::string request_type);Function：Get kernel version information Parameter Definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：gc.getCoreVersion(); - -**The API version information is displaye: getAPIVersion** - - std::string getAPIVersion(std::string request_type); - Function：Get the API version information - Parameter Definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：gc.getAPIVersion(); - -**Query the database and save the file: fquery** - - void fquery(std::string db_name, std::string format, std::string sparql, std::string filename, std::string request_type); - Function：Query the database and save the results to a file - Parameter Definition：[database name], [query result type JSON, HTML or text], [SPARQL statement], [file name], [request type "GET" and "POST ", if the request type is "GET", it can be omitted] - Example：gc.fquery("lubm", "json", sparql, "ans.txt"); - -**Exporting the Database** - - std::string exportDB(std::string db_name, std::string dir_path, std::string request_type); - Function：Export the database to a folder - Parameter Definition：[database name], [path to database export folder], [request type "GET" and "POST ", if the request type is "GET", can be omitted] - Example：gc.exportDB("lubm", "/root/gStore/"); - - - -
- -### Java HTTP API - ---- - -To use the Java API, please refer to the gStore/API/HTTP/Java/SRC/JGSC/GstoreConnector. Java. Specific use is as follows: - -**Construct the initialization function** - - public class GstoreConnector(String serverIP, int serverPort, String username, String password); - Function：Initialize - Parameter Definition：[Server IP], [GHTTP port on the server], [Username], [password] - Example：GstoreConnector gc = new GstoreConnector("127.0.0.1", 9000, "root", "123456"); - -**Building a database: build** - - public String build(String db_name, String rdf_file_path, String request_type); - Function：Create a new database from an RDF file - Parameter Definition：[database name], [.nt file path], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：gc.build("lubm", "data/lubm/lubm.nt"); - -**Loading a database: load** - - public String load(String db_name, String request_type); - Function：Load the database you created - Parameter Definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：gc.load("lubm"); - -**Stopping database loading: unload** - - public String unload(String db_name, String request_type); - Function：Stopping database loading - Parameter Definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：gc.unload("lubm"); - -**User management：user** - - public String user(String type, String username2, String addition, String request_type); - Function：The root user can add, delete, or modify the user's permission only. - 1.Add or delete users： - Parameter Definition：["add_user" adds user, "delete_user" deletes user],[user name],[password],[request type "GET" and "POST ", if request type "GET" can be omitted] - Example：gc.user("add_user", "user1", "111111"); - 2.Modify user's privilege： - Parameter Definition：["Add_query" adds query permission, "delete_query" deletes query permission, "add_load" adds load permission, "delete_load" deletes load permission, "add_unload" adds no load permission, "delete_unload" deletes no load permission, "Add_update" adds update permission, "delete_update" deletes update permission, "add_backup" adds backup permission, "delete_bakup" deletes backup permission, "add_restore" adds restore permission, "Delete_restore" deletes restore permission,"add_export" adds export permission," delete_export" deletes export permission],[user name],[database name],[Request type "GET" and "POST ", if the request type is "GET", it can be omitted.] - Example：gc.user("add_query", "user1", "lubm"); - -**Display user：showUser** - - public String showUser(String request_type); - Function：Display all users - Parameter Definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：gc.showUser(); - -**Database query: query** - - public String query(String db_name, String format, String sparql, String request_type); - Function：query databse - Parameter Definition：[database name], [query result type JSON, HTML or text], [SPARQL statement], [Request type "GET" and "POST ", if the request type is "GET", it can be omitted] - Example： - String res = gc.query("lubm", "json", sparql); - System.out.println(res); //output result - -**Database deletion ：drop** - - public String drop(String db_name, boolean is_backup, String request_type); - Function：Delete the database directly or delete the database while leaving a backup. - Parameter Definition：[database name], [false not backup, true backup], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：gc.drop("lubm", false); //Delete the database without leaving a backup - -**Monitoring database: monitor** - - public String monitor(String db_name, String request_type); - Function：Displays information for a specific database - Parameter Definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted]Example：gc.monitor("lubm"); - -**Save the database: checkpoint** - - public String checkpoint(String db_name, String request_type); - Function：If you change the database, save the database - Parameter Definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted]Example：gc.checkpoint("lubm"); - -**Show database：show** - - public String show(String request_type);Function：Displays all created databasesParameter Definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"]Example：gc.show(); - -**The kernel version information is displayed：getCoreVersion** - - public String getCoreVersion(String request_type);Function：Get kernel version information Parameter Definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"]Example：gc.getCoreVersion(); - -**Display API version：getAPIVersion** - - public String getAPIVersion(String request_type);Function：Get API version Parameter：[Request types "GET" and "POST" can be omitted if the request type is "GET"]Example：gc.getAPIVersion(); - -**Query the database and save the file: fquery** - - public void fquery(String db_name, String format, String sparql, String filename, String request_type);Function：Query the database and save the result to a file Parameter definition：[database name], [query result type JSON, HTML or text], [SPARQL statement], [file name]，[Request types "GET" and "POST" can be omitted if the request type is "GET"]Example：gc.fquery("lubm", "json", sparql, "ans.txt"); - -**Export database** - - public String exportDB(String db_name, String dir_path, String request_type);Function：Export database to parameter definition under file folder：[database name], [database export folder path]，[Request types "GET" and "POST" can be omitted if the request type is "GET"】Example：gc.exportDB("lubm", "/root/gStore/"); - -
- -### Python HTTP API - ---- - -To use the Python API, please refer to the gStore/API/HTTP/Python/SRC/GstoreConnector. Py. Specific use is as follows: - -**Construct the initialization function** - - public class GstoreConnector(self, serverIP, serverPort, username, password): - Function：Initialization - Parameter definition：[Server IP], [GHTTP port on the server], [Username], [password]. - Example：gc = GstoreConnector.GstoreConnector("127.0.0.1", 9000, "root", "123456") - -**Build database: build** - - def build(self, db_name, rdf_file_path, request_type): - Function：Create a new database from an RDF file - Parameter definition：[Database name]，[.nt文件路径]，[Request types "GET" and "POST" can be omitted if the request type is "GET"】 - Example：res = gc.build("lubm", "data/lubm/lubm.nt") - -**Load database: load** - - def load(self, db_name, request_type): - Function：load the database you have created - Parameter definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：res = gc.load("lubm") - -**Unload database: unload** - - def unload(self, db_name, request_type): - Function：Unload database - Parameter definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：res = gc.unload("lubm") - -**User management: user** - - def user(self, type, username2, addition, request_type): - Function：The root user can add, delete, or modify the user's permission only. - 1.Add or delete users： - Parameter definition：["add_user" adds a user, "delete_user" deletes a user], [username],[password],[Request types "GET" and "POST" can be omitted if the Request type is "GET"] - Example：res = gc.user("add_user", "user1", "111111") - 2.Modify user's privilege： - Parameter definition：["Add_query" adds query permission, "delete_query" deletes query permission, "add_load" adds load permission, "delete_load" deletes load permission, "add_unload" adds no load permission, "delete_unload" deletes no load permission, "Add_update" adds update permission, "delete_update" deletes update permission, "add_backup" adds backup permission, "delete_bakup" deletes backup permission, "add_restore" adds restore permission, "Delete_restore" deletes restore permission,"add_export" adds export permission," delete_export" deletes export permission], [user name],[database name],[Request types "GET" and "POST" can be omitted if the Request type is "GET"] - Example：res = gc.user("add_query", "user1", "lubm") - -**Display users: showUser** - - def showUser(self, request_type): - Function：Display all users - Parameter definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.showUser() - -**Query Database: query** - - def query(self, db_name, format, sparql, request_type): - Function：Query the database - Parameter definition：[Database name]，[Query result type JSON, HTML or text], [SPARQL statement], [Request types "GET" and "POST" can be omitted if the Request type is "GET"] - Example： - res = gc.query("lubm", "json", sparql) - print(res) //output result - -**Database deletion: drop** - - def drop(self, db_name, is_backup, request_type): - Function：Delete the database directly or delete the database while leaving a backup - Parameter definition：[database name], [false not backup, true backup],[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.drop("lubm", false) //Delete the database without leaving a backup - -**Database Monitor ：monitor** - - def monitor(self, db_name, request_type): Function：Displays information for a specific database Parameter definition：[Database name]，[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.monitor("lubm") - -**Save database: checkpoint** - - def checkpoint(self, db_name, request_type):Function：If the database is changed, save the meaning of the database parameters：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] Example：res = gc.checkpoint("lubm") - -**Display database：show** - - def show(self, request_type):Function：Display the meanings of all created databases：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.show() - -**The kernel version information is displayed: getCoreVersion** - - def getCoreVersion(self, request_type):Function：Get the definition of kernel version parameter ：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.getCoreVersion() - -**Display API version: getAPIVersion** - - def getAPIVersion(self, request_type):Function：Get the API version information parameter definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.getAPIVersion() - -**Query the database and save the file: fquery** - - def fquery(self, db_name, format, sparql, filename, request_type):Function：Query the database and save the result to a file Parameter definition：[database name], [query result type JSON, HTML or text], [SPARQL statement], [file name]，[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：gc.fquery("lubm", "json", sparql, "ans.txt") - -**Export databse** - - def exportDB(self, db_name, dir_path, request_type): Function：parameter definition of exprotin database to folders：[database name], [database export folder path]，[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.exportDB("lubm", "/root/gStore/") - -
- -### Nodejs HTTP API - ---- - -Before using the Nodejs API, type `NPM install Request` and `NPM Install request-promise` to add the required modules under the Nodejs folder. - -To use Nodejs API, please refer to the `gStore/API/http/Nodejs/GstoreConnector.js`. Specific use is as follows: - -**Construct the initialization function** - - class GstoreConnector(ip = '', port, username = '', password = ''); - Function：Initialization - Parameter definition：[Server IP], [GHTTP port on the server], [Username], [password] - Example：gc = new GstoreConnector("127.0.0.1", 9000, "root", "123456"); - -**Build database: build** - - async build(db_name = '', rdf_file_path = '', request_type); - Function：Create a new database from an RDF file - The defintion of parameters are as follows: [database name], [.nt file path],[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.build("lubm", "data/lubm/lubm.nt"); - -**Load database：load** - - async load(db_name = '', request_type); - Function：Load the database you have created - Parameter definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：res = gc.load("lubm"); - -**Unload databse：unload** - - async unload(db_name = '', request_type); - Function：Unload databse - Parameter definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：res = gc.unload("lubm"); - -**User management：user** - - async user(type = '', username2 = '' , addition = '' , request_type); - Function：The root user can add, delete, or modify the user's permission only - 1.Add or delete users： - Parameter definition：["add_user" adds a user, "delete_user" deletes a user], [user name],[password],[Request types "GET" and "POST" can be omitted if the Request type is "GET"] - Example：res = gc.user("add_user", "user1", "111111"); - 2.Privilege to modify user： - Parameter definition：["Add_query" adds query permission, "delete_query" deletes query permission, "add_load" adds load permission, "delete_load" deletes load permission, "add_unload" adds no load permission, "delete_unload" deletes no load permission, "Add_update" adds update permission, "delete_update" deletes update permission, "add_backup" adds backup permission, "delete_bakup" deletes backup permission, "add_restore" adds restore permission, "Delete_restore" deletes restore permission,"add_export" adds export permission," delete_export" deletes export permission], [user name],[database name],[Request types "GET" and "POST" can be omitted if the Request type is "GET"]] - Example：res = gc.user("add_query", "user1", "lubm"); - -**Display user：showUser** - - async showUser(request_type); - Function：Display all users - Parameter definition: [Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.showUser(); - -**Query database：query** - - async query(db_name = '', format = '' , sparql = '' , request_type); - Function：Query database - Parameter definition：[Database name], [query result type JSON, HTML or text], [SPARQL statement], [Request types "GET" and "POST" can be omitted if the Request type is "GET"] - Example： - res = gc.query("lubm", "json", sparql); - console.log(JSON.stringify(res,",")); //output result - -**Database deletion：drop** - - async drop(db_name = '', is_backup , request_type); - Function：Delete the database directly or delete the database while leaving a backup - Parameter definition：[database name]，[false no backup, true backup]，[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.drop("lubm", false); //Delete the database without leaving a backup - -**Database monitor: monitor** - - async monitor(db_name = '', request_type); Function：Parameter definition for displaying information about a specific database：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted]Example：res = gc.monitor("lubm"); - -**Save database: checkpoint** - - async checkpoint(db_name = '', request_type);Function：If the database is changed, the parameter definition of saving database：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted]Example：res = gc.checkpoint("lubm"); - -**Display database: show** - - async show(request_type);Function：Displays all created databases Parameter definition: [Request types "GET" and "POST" can be omitted if the request type is "GET"]Example：res = gc.show(); - -**Display kernel version information: getCoreVersion** - - async getCoreVersion(request_type);Function：Get kernel version information - Parameter definition: [Request types "GET" and "POST" can be omitted if the request type is "GET"]Example：res = gc.getCoreVersion(); - -**Display API version: getAPIVersion** - - async getAPIVersion(request_type); - Function：Get the API version information - Parameter definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：res = gc.getAPIVersion(); - -**Query the database and save the file: fquery** - - async fquery(db_name = '', format = '' , sparql = '' , filename = '' , request_type); - Function：Query the database and save the results to a file - Parameter definition：[database name], [query result type JSON, HTML or text], [SPARQL statement], [file name], [Request types "GET" and "POST" can be omitted if the Request type is "GET"] - Example：gc.fquery("lubm", "json", sparql, "ans.txt"); - -**Export database** - - async exportDB(db_name = '' , dir_path = '' , request_type); - Function：export database to folders - parameter definition：[database name], [directory where the database is exported], [Request types "GET" and "POST" can be omitted if the Request type is "GET"] - Example：res = gc.exportDB("lubm", "/root/gStore/"); - -
- -### PHP HTTP API - ---- - -To use the Php API, please refer to the gStore/API/HTTP/Php/SRC/GstoreConnector. Php. Specific use is as follows: - -**Construct the initialization function** - - class GstoreConnector($ip, $port, $username, $password) - Function：Initialization - Parameter definition：[Server IP], [GHTTP port on the server], [Username], [password] - Example：$gc = new GstoreConnector("127.0.0.1", 9000, "root", "123456"); - -**Build databse：build** - - function build($db_name, $rdf_file_path, $request_type) - Function：Create a new database from an RDF file - Parameter definition：[database name], [.nt file path],[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example： - $res = $gc->build("lubm", "data/lubm/lubm.nt"); - echo $res . PHP_EOL; - -**Load database：load** - - function load($db_name, $request_type) - Function：Load the database you have created - Parameter definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example： - $ret = $gc->load("test"); - echo $ret . PHP_EOL; - -**Unload databse：unload** - - function unload($db_name, $request_type) - Function：Unload databse - Parameter definition：[database name]，[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example： - $ret = $gc->unload("test"); - echo $ret . PHP_EOL; - -**User management：user** - - function user($type, $username2, $addition, $request_type) - Function：Only the root user can add, delete, or modify the user's permission - 1.Add or delete users： - Parameter definition：["add_user" adds a user, "delete_user" deletes a user], [user name],[password],[Request types "GET" and "POST" can be omitted if the Request type is "GET"] - Example： - $res = $gc->user("add_user", "user1", "111111"); - echo $res . PHP_EOL; - 2.Privilege to modify user： - 参数含义：["Add_query" adds query permission, "delete_query" deletes query permission, "add_load" adds load permission, "delete_load" deletes load permission, "add_unload" adds no load permission, "delete_unload" deletes no load permission, "Add_update" adds update permission, "delete_update" deletes update permission, "add_backup" adds backup permission, "delete_bakup" deletes backup permission, "add_restore" adds restore permission, "Delete_restore" deletes restore permission,"add_export" adds export permission," delete_export" deletes export permission], [user name],[database name],[Request types "GET" and "POST" can be omitted if the Request type is "GET"]] - Example： - $res = $gc->user("add_user", "user1", "lubm"); - echo $res . PHP_EOL; - -**Display user：showUser** - - function showUser($request_type) - Function：Display all users - Parameter definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example： - $res = $gc->showUser(); - echo $res. PHP_EOL; - -**Query database：query** - - function query($db_name, $format, $sparql, $request_type) - Parameter definition：[database name], [query result type JSON, HTML or text], [SPARQL statement],[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example： - $res = $gc->query("lubm", "json", $sparql); - echo $res. PHP_EOL; //output result - -**Database deletion：drop** - - function drop($db_name, $is_backup, $request_type) - Function：Delete the database directly or delete the database while leaving a backup - Parameter definition：[database name], [false not backup, true backup],[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example： - $res = $gc->drop("lubm", false); //Delete the database without leaving a backup - echo $res. PHP_EOL; - -**Database monitor: monitor** - - function monitor($db_name, $request_type)Function：Displays information for a specific database - Parameter definition：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted] - Example：$res = $gc->monitor("lubm");echo $res. PHP_EOL; - -**Save database: checkpoint** - - function checkpoint($db_name, $request_type)Function：the Parameter definition of save database if it has been changed：[database name], [request type "GET" and "POST ", if request type "GET" can be omitted]Example：$res = $gc->checkpoint("lubm");echo $res. PHP_EOL; - -**Display database: show** - - function show($request_type)Function：Displays all created databases - Parameter definition：[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：$res = $gc->show();echo $res. PHP_EOL; - -**Display kernel version information: getCoreVersion** - - function getCoreVersion($request_type)Function：get kernel version information - Parameter definition: [Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：$res = $gc->getCoreVersion();echo $res. PHP_EOL; - -**Display API version: getAPIVersion** - - function getAPIVersion($request_type) - Function：Get API version - Parameter definition: [Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example： - $res = $gc->getAPIVersion(); - echo $res. PHP_EOL; - -**Query the database and save the file：fquery** - - function fquery($db_name, $format, $sparql, $filename, $request_type) - Function：Query the database and save the results to a file - Parameter definition：[database name], [query result type JSON, HTML or text], [SPARQL statement], [file name],[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：$gc->fquery("lubm", "json", $sparql, "ans.txt"); - -**Export database** - - function exportDB($db_name, $dir_path, $request_type) - Function：Export the database to a folder - Parameter definition：[database name], [database export folder path]，[Request types "GET" and "POST" can be omitted if the request type is "GET"] - Example：$res = $gc->exportDB("lubm", "/root/gStore/"); - -
- -### gServer API instruction - ---- - -#### API Interconnection Mode - -> The gServer interface uses the `socket` protocol and supports multiple ways to access the interface. If the port `9000` is started from the gServer in the Main directory, the contents of the interface interconnection are as follows: -> -> API address： -> -> ```json -> http://ip:9000/ -> ``` -> -> The API supports the input of a parameter list in JSON format, as shown below： -> -> ```json -> {"op": "[op_type]", "[paramname1]": "[paramvalue1]", "[paramname2]": "[paramvalue2]"……} -> ``` -> -> - - - -#### API List - -| API name | Definition | Note | -| -------- | ------------------------------ | ------------------------------------------------------- | -| build | Build graph database | The database file must be locally stored on the server | -| load | Load graph database | Load the database into memory | -| unload | Unload graph database | Unload the database from memory | -| drop | Delete graph database | Logical deletion and physical deletion can be performed | -| show | Display graph database | Display list of all databases | -| query | Query graph database | Including query, delete, and insert | -| stop | Close server | Only root user root can perform this operation | -| close | Close client server connection | Process client connection closure requests | -| login | login to database | authenticate user names and password | -| | | | - -
- -#### API specific instruction - -> This section describes the input and output parameters of each interface. Assume that the IP address of the gserver is 127.0.0.1 and the port is 9000 - -##### 5.9.3.1 build- build database - -###### Brief description - -- Create a database based on existing NT file -- Files must exist on the gStore server - -###### Request ip - -- ` 127.0.0.1 ` - -###### Request port number - -- ` 9000 ` - -###### Parameter transfer mode - -- Pass it as a `JSON` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ------------------------------------------------------------ | -| op | yes | string | Operation name, fixed value is**build** | -| db_name | yes | string | Database name (.db is not required) | -| db_path | yes | string | Database file path (can be an absolute path or a relative path. The relative path uses the gStore installation root directory as a reference directory) | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "Import RDF file to database done." -} -``` - -
- -##### load - -###### Brief description - -- To load a database into memory, a load operation is a prerequisite for many operations, such as Query - -###### Request ip - -- ` 127.0.0.1 ` - -###### Request port number - -- ` 9000 ` - -###### Parameter transfer mode - -- Pass it as a `JSON` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | -------------------------------------- | -| op | yes | string | Operation name, fixed value is**load** | -| db_name | yes | string | Database name (.db is not required) | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "Load database successfully." -} -``` - -
- -##### unload - -###### Brief description - -- Unmount the database from memory (all changes are flushed back to hard disk) - -###### Request ip - -- ` 127.0.0.1 ` - -###### Request port number - -- ` 9000 ` - -###### Parameter transfer mode - -- Pass it as a `JSON` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ---------------------------------------- | -| op | yes | string | Operation name, fixed value is**unload** | -| db_name | yes | string | Database name (.db is not required) | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "Unload database done." -} -``` - -
- -##### drop - -###### Brief description - -- Delete the database - -###### Request ip - -- ` 127.0.0.1 ` - -###### Request port number - -- ` 9000 ` - -###### Parameter transfer mode - -- Pass it as a `JSON` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | -------------------------------------- | -| op | yes | string | Operation name, fixed value is**drop** | -| db_name | yes | string | Database name (.db is not required) | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "Drop database done." -} -``` - -
- -##### show - -###### Brief description - -- Display all database list - -###### Request ip - -- ` 127.0.0.1 ` - -###### Request port number - -- ` 9000 ` - -###### Parameter transfer mode - -- Pass it as a `JSON` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | -------------------------------------- | -| op | yes | string | Operation name, fixed value is**show** | - -###### Return value - -| Parameter name | Type | Note | -| :---------------- | :-------- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| ResponseBody | JSONArray | JSON arrays (each of which is a database information0 | -| -------- database | string | database name | -| ---------status | string | database status | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "success", - "ResponseBody": [ - "lubm": "loaded", - "lubm10K": "unloaded" - ] -} -``` - -
- -##### query - -###### Brief description - -- query database - -###### Request ip - -- ` 127.0.0.1 ` - -###### Request port number - -- ` 9000 ` - -###### Parameter transfer mode - -- Pass it as a `JSON` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | ----------------------------------------------- | -| op | yes | string | Operation name, fixed value is**query** | -| db_name | yes | string | database that need operations | -| format | no | string | The result set return format is json by default | -| sparql | yes | string | The SPARQL statement to execute | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | -| head | JSON | Head information | -| results | JSON | Result information (see Return Sample for details) | - - -###### Return sample - -``` json -{ - "head": { - "link": [], - "vars": [ - "x" - ] - }, - "results": { - "bindings": [ - { - "x": { - "type": "uri", - "value": "十面埋伏" - } - }, - { - "x": { - "type": "uri", - "value": "投名状" - } - }, - { - "x": { - "type": "uri", - "value": "如花" - } - } - ] - }, - "StatusCode": 0, - "StatusMsg": "success" -} -``` - -
- -##### login - -###### Brief description - -- Login user (verify username and password) - -###### Request ip - -- ` 127.0.0.1 ` - -###### Request port number - -- ` 9000 ` - -###### Parameter transfer mode - -- Pass it as a `JSON` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | --------------------------------------- | -| op | yes | string | Operation name, fixed value is**login** | -| username | yes | string | user name | -| password | yes | string | Password (plain text) | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ - "StatusCode": 1001, - "StatusMsg": "wrong password." -} -``` - -
- -##### stop - -###### Brief description - -- Close server - -###### Request ip - -- ` 127.0.0.1 ` - -###### Request port number - -- ` 9000 ` - -###### Parameter transfer mode - -- Pass it as a `JSON` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | -------------------------------------- | -| op | yes | string | Operation name, fixed value is**stop** | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "Server stopped." -} -``` - -
- -##### close - -###### Brief description - -- Close the connection to the client - -###### Request ip - -- ` 127.0.0.1 ` - -###### Request port number - -- ` 9000 ` - -###### Parameter transfer mode - -- Pass it as a `JSON` structure - -###### Parameter - -| Parameter name | Mandatory | Type | Note | -| :------------- | :-------- | :----- | --------------------------------------- | -| op | yes | string | Operation name, fixed value is**close** | - -###### Return value - -| Parameter name | Type | Note | -| :------------- | :----- | ------------------------------------------------------------ | -| StatusCode | int | Return value code value (refer to attached table: Return value code table for details) | -| StatusMsg | string | Return specific information | - - -###### Return sample - -``` json -{ - "StatusCode": 0, - "StatusMsg": "Connection disconnected." -} -``` +#### Run -
- -##### Attached table 1 return value code table - -| Code value | Definition | -| ---------- | ------------------------------------------- | -| 0 | Success | -| 1000 | The method type is not support | -| 1001 | Authentication Failed | -| 1002 | Check Privilege Failed | -| 1003 | Param is illegal | -| 1004 | The operation conditions are not satisfied | -| 1005 | Operation failed | -| 1006 | Add privilege Failed | -| 1007 | Loss of lock | -| 1008 | Transcation manage Failed | -| 1100 | The operation is not defined | -| 1101 | IP Blocked | -| | | +You are advised to see gStore/api/socket/python/example/PythonAPIExample for examples on how to use python API. Python file does not need compiling, and you can run it directly. diff --git a/docs/DEMAND.md b/docs/DEMAND.md index dc96d161..b3d90a60 100644 --- a/docs/DEMAND.md +++ b/docs/DEMAND.md @@ -1,30 +1,27 @@ -*We have tested on linux server with CentOS 6.2 x86_64 and CentOS 6.6 x86_64. The version of GCC should be 4.8 or later.The version of boost should be 1.56 or later* - -| Project | Requirement | -| :--------------- | :----------------------------------------------------- | -| Operating System | Linux, such as CentOS, Ubuntu etc. | -| Framework | x86_64 | -| Disk Size | Depends on the size of the data set | -| Memory Size | Depends on the size of the data set | -| glibc | Must install version >= 2.14 | -| gcc | Must install version >= 5.0 | -| g++ | Must install version >= 5.0 | -| make | Must install | -| cmake | Must install | -| pkg-config | Must install | -| uuid | Must install | -| boost | Must install version >= 1.56 && <= 1.59 | -| readline | Must install | -| readline-devel | Must install | -| libcurl-devel | Must install | -| openjdk | If using the Java API, yes | -| openjdk-devel | If using the Java API, yes | -| requests | If using Python http API, yes | -| node | If using Nodejs http api must install version >=10.9.0 | -| curl-devel | If using php http API, yes | -| pthreads | If using php http API, yes | -| realpath | If using gconsole, yes | -| ccache | Optional, used to speed up compilation | +*We have tested on linux server with CentOS 6.2 x86_64 and CentOS 6.6 x86_64. The version of GCC should be 4.8 or later.* + +Item | Requirement +:-- | :-- +operation system | Linux, such as CentOS, Ubuntu and so on +architecture | x86_64 +disk size | according to size of dataset +memory size | according to size of dataset +glibc | version >= 2.14 +gcc | version >= 4.8 +g++ | version >= 4.8 +make | need to be installed +boost | version >= 1.54 +readline | need to be installed +readline-devel | need to be installed +openjdk | needed if using Java api +openjdk-devel | needed if using Java api +requests | needed if using Python http api +node | needed if using Nodejs http api and version >= 10.9.0 +pthreads | needed if using php http api +curl-devel | needed if using php http api +realpath | needed if using gconsole +ccache | optional, used to speed up the compilation +libcurl-devel | needed to be installed NOTICE: @@ -44,7 +41,7 @@ NOTICE: 7. To install ccache, you need to add epel repository if using CentOS, while in Ubuntu you can directly install it by `apt-get install ccache` command. If you can not install ccache(or maybe you do not want to), please go to modify the makefile(just change the CC variable to g++). -8. If you need to use the HTTP server in gStore, then Boost Library(like boost-devel, including boost headers for developing) must be installed and the version should not be less than 1.56. Remember to check the makefile for your installed path of Boost. To use Python api, you need to install requests by `pip install requests` in CentOS. To use php api, you need to install pthreads and curl in CentOS as follows: +8. If you need to use the HTTP server in gStore, then Boost Library(like boost-devel, including boost headers for developing) must be installed and the version should not be less than 1.54. Remember to check the makefile for your installed path of Boost. To use Python api, you need to install requests by `pip install requests` in CentOS. To use php api, you need to install pthreads and curl in CentOS as follows: ``` 1- get curl-devel # yum install curl-devel diff --git a/docs/DOCKER_DEPLOY_CN.md b/docs/DOCKER_DEPLOY_CN.md index 5eba6b43..0fc09c15 100644 --- a/docs/DOCKER_DEPLOY_CN.md +++ b/docs/DOCKER_DEPLOY_CN.md @@ -12,26 +12,33 @@ 关于安装使用Docker，官方针对常见Linux发行版文档已经写得很详细，就直接给出参考地址：[英文文档](https://docs.docker.com/install/linux/docker-ce/ubuntu/)，[中文文档](https://docs.docker-cn.com/engine/installation/linux/docker-ce/centos/#%E5%85%88%E5%86%B3%E6%9D%A1%E4%BB%B6) 。 +需要注意的是，Docker版本过高可能导致一些问题，建议仔细阅读注意事项。当前测试环境版本是Docker CE 18.06.1 + ## 0x01.通过Dockerfile构建镜像 -假设已经拥有正常的Docker环境跟网络后，首先通过 `git clone` 下载项目。然后进入项目根目录，输入命令`docker build -t gstore .` 即可开始构建。关于具体说明请参见 Dockerfile 文件内容。 +假设已经拥有正常的Docker环境跟网络后，首先通过`git clone ` 下载项目。然后进入项目根目录，输入命令`make pre `解压一些工具包。输入命令`docker build -t gstore .` 即可开始构建，默认使用根目录的Dockerfile，关于具体说明Dockerfile文件内也有补充。 -构建完成后，直接通过 `docker run -p 9000:80 -it gstore` 即可启动并进入容器执行其他操作。 +构建完成后，直接通过`docker run -p 9000:80 -it gstore` 即可启动并进入容器执行其他操作。 ## 0x02.直接拉取镜像运行(推荐) -无需下载项目或自己构建，直接输入 `docker pull pkumodlab/gstore:latest` 拉取已经在 docker hub 上自动构建完成的镜像。拉取完成后 `docker run -p 9000:80 -it pkumodlab/gstore:latest` 即可直接启动并进入容器使用。 +无需下载项目或自己构建，直接输入`docker pull pkumod/gstore:latest` 拉取已经在docker hub上自动构建完成的镜像。拉取完成后 `docker run -p 9000:80 -it pkumod/gstore:latest ` 即可直接启动并进入容器使用。 ## 0x03.后续工作 #### A.性能测试 -这个待后续补充，容器化跟原生运行gStore，在不同文件数/网络等情况下，容器的性能损耗具体比重。 +这个待后续补充，容器化跟原生运行gStore，在不同文件数/网络等情况下，容器的性能损耗具体比重。。 #### B.连接其他容器测试 文档也待后续补充更新 +#### C.构建精简优化 + +因为gcc:8镜像就有1.7G了，自带了很多不必要的东西（包括go的环境之类的），希望后续除了在调低gcc版本之外，能对源镜像本身改进出精简版来。 + --- 其他可能也有不少需要补充，所以目前只是**抛砖引玉**，添加了一个最基本的版本。基本的环境构建只是容器化的第一步，后续工作还不少~ 这只是初版的V1.0文档。 + diff --git a/docs/DOCKER_DEPLOY_EN.md b/docs/DOCKER_DEPLOY_EN.md index 2f318424..6d9d2814 100644 --- a/docs/DOCKER_DEPLOY_EN.md +++ b/docs/DOCKER_DEPLOY_EN.md @@ -1,26 +1,28 @@ # Deploy gStore by Docker -[点击查看中文文档](DOCKER_DEPLOY_CN.md) +[点击查看中文文档](Docker方式部署gStore.md) >Roughly speaking, there are two ways to deploy gStore via Docker. > ->The first one is using the Dockerfile file in the root directory of the project to automatically build, and then run the container. +>The first one is using Dockerfile file in the root directory of project to automatically build it. And then run the container. > ->Another one is downloading the pre-built image, and running it directly. +>Another one is downloading the mirror which has been automatically built directly, then just run it. ## 0x00. prepare the environment -Official doc of Docker has explained how to download and use it on the common Liunx release version in detail. And here is the link: [English doc](https://docs.docker.com/install/linux/docker-ce/ubuntu/), [中文文档](https://docs.docker-cn.com/engine/installation/linux/docker-ce/centos/#%E5%85%88%E5%86%B3%E6%9D%A1%E4%BB%B6) +Official doc of Docker has explained how to download and use it on common Liunx release version in details. And here is the link: [English doc](https://docs.docker.com/install/linux/docker-ce/ubuntu/), [中文文档](https://docs.docker-cn.com/engine/installation/linux/docker-ce/centos/#%E5%85%88%E5%86%B3%E6%9D%A1%E4%BB%B6) -## 0x01. Build the image via Dockerfile +It's worth noting that the Docker with too high version may lead to some problems. Please read the precautions carefully. The current version of test environment is Docker CE 18.06.1 -After having the correct Docker environment and network, use `git clone` to download the project firstly. Then enter the root directory and input the command `docker build -t gstore .` to start building. The more specific explanation has been written in the Dockerfile. +## 0x01. Build the mirror via Dockerfile + +After having the correct Docker environment and network, use `git clone` to download the project firstly. Then enter the root directory and input command `make pre ` to unzip some toolkits. After inputting command `docker build -t gstore` it's available to start building. In the default case, it will use the Dockerfile in the root directory. More specific explanation has been written in the Dockerfile. After the building, using `docker run -p 9000:80 -it gstore` directly to start and enter the container and execute other operations. -## 0x02. pulling the pre-built image directly to run(recommendation) +## 0x02. pulling the mirror directly to run(recommendation) -Instead of downloading this project or building by yourself, input `docker pull pkumodlab/gstore:latest` to pull the docker image which has been built well on the docker hub. Then input `docker run -p 9000:80 -it pkumodlab/gstore:latest` to start and enter the container and execute other operations. +Instead of downloading project or building on your own, input `docker pull pkumod/gstore:latest` to pull the mirror which has been automatically built well on the docker hub. Then input `docker run -p 9000:80 -it pkumod/gstore:latest` to start and enter the container and execute other operations. ## 0x03. Follow-up @@ -34,6 +36,10 @@ It's waiting for supplement. Waiting for supplement. +### C. Simplification and optimization of building + +The mirror of gcc:8 has conquered the space of 1.7G, bringing a lot of unnecessary things(including the environment of Go). Hoping to optimize the mirror source in the future excepting lowing the gcc's version. + --- There are still a large number of content waiting for supplement. diff --git a/makefile b/makefile index 39978ac3..006b7611 100644 --- a/makefile +++ b/makefile @@ -54,7 +54,8 @@ EXEFLAG = -O2 -pthread -std=c++11 #library = -lreadline -L./lib -L/usr/local/lib -L/usr/lib/ -lantlr4-runtime -lgcov -lboost_thread -lboost_filesystem -lboost_system -lboost_regex -lpthread -I/usr/local/include/boost -lcurl -llog4cplus -Wl,-rpath='/usr/local/lib' library = -lreadline -L./lib -L/usr/local/lib -L/usr/lib/ -lantlr4-runtime -lgcov -lboost_thread -lboost_filesystem -lboost_system -lboost_regex -lpthread -I/usr/local/include/boost -lcurl #used for parallelsort -openmp = -fopenmp -march=native +march = -march=native +openmp = -fopenmp ${march} # library = -ltermcap -lreadline -L./lib -lantlr -lgcov def64IO = -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_LARGEFILE_SOURCE -- Gitee