From 513e3156d44ffe91b14807ffb2bca30611e25418 Mon Sep 17 00:00:00 2001 From: Qiheng Lin Date: Wed, 26 Apr 2023 23:05:14 +0800 Subject: [PATCH 1/3] dentry: LoadChildren and MetaFileMgr Signed-off-by: Qiheng Lin Change-Id: Id362abb99afdbb8e899fcdd5bfb54e46785b1731 --- .../dentry/dentry_meta_file_test.cpp | 15 ++ utils/dentry/include/meta_file.h | 28 +++- utils/dentry/src/file_utils.cpp | 8 +- utils/dentry/src/meta_file.cpp | 137 +++++++++++++++++- 4 files changed, 176 insertions(+), 12 deletions(-) diff --git a/test/unittests/cloudsync_sa/dentry/dentry_meta_file_test.cpp b/test/unittests/cloudsync_sa/dentry/dentry_meta_file_test.cpp index 64508d476..6408cb2b6 100644 --- a/test/unittests/cloudsync_sa/dentry/dentry_meta_file_test.cpp +++ b/test/unittests/cloudsync_sa/dentry/dentry_meta_file_test.cpp @@ -146,4 +146,19 @@ HWTEST_F(DentryMetaFileTest, MetaFileRemove, TestSize.Level1) ret = mFile.DoLookup(mBase2); EXPECT_EQ(ret, ENOENT); } + +/** + * @tc.name: MetaFileMgr + * @tc.desc: Verify the MetaFileMgr + * @tc.type: FUNC + * @tc.require: SR000HRKJB + */ +HWTEST_F(DentryMetaFileTest, MetaFileMgr, TestSize.Level1) +{ + auto m = MetaFileMgr::GetInstance().GetMetaFile(TEST_USER_ID, "/o/p/q/r/s/t"); + MetaBase mBase1("file1", "file1"); + EXPECT_EQ(m->DoCreate(mBase1), 0); + m = nullptr; + MetaFileMgr::GetInstance().ClearAll(); +} } // namespace OHOS::FileManagement::CloudSync::Test diff --git a/utils/dentry/include/meta_file.h b/utils/dentry/include/meta_file.h index c81dc7192..47f8dbf74 100644 --- a/utils/dentry/include/meta_file.h +++ b/utils/dentry/include/meta_file.h @@ -16,8 +16,10 @@ #ifndef OHOS_FILEMGMT_DENTRY_META_FILE_H #define OHOS_FILEMGMT_DENTRY_META_FILE_H +#include #include #include +#include #include #include #include @@ -39,6 +41,7 @@ public: int32_t DoUpdate(const MetaBase &base); int32_t DoRename(const MetaBase &oldBase, const std::string &newName); int32_t DoLookup(MetaBase &base); + int32_t LoadChildren(std::vector &bases); private: std::mutex mtx_{}; @@ -49,6 +52,24 @@ private: std::shared_ptr parentMetaFile_{nullptr}; }; +class MetaFileMgr { +public: + static MetaFileMgr& GetInstance() { + static MetaFileMgr instance_; + return instance_; + } + std::shared_ptr GetMetaFile(uint32_t userId, const std::string &path); + void ClearAll(); +private: + MetaFileMgr() = default; + ~MetaFileMgr() = default; + MetaFileMgr(const MetaFileMgr &m) = delete; + const MetaFileMgr &operator=(const MetaFileMgr &m) = delete; + + std::recursive_mutex mtx_{}; + std::map, std::shared_ptr> metaFiles_; +}; + struct MetaBase { MetaBase(const std::string &name) : name(name) {} MetaBase(const std::string &name, const std::string &cloudId) : name(name), cloudId(cloudId) {} @@ -58,9 +79,12 @@ struct MetaBase { uint32_t mode{S_IFREG}; std::string name{}; std::string cloudId{}; +}; - int refcnt{0}; - std::mutex refMtx; +struct MetaInode { + MetaBase mBase; + std::atomic refcnt{0}; + std::shared_ptr parentMetaFile_{nullptr}; }; struct BitOps { diff --git a/utils/dentry/src/file_utils.cpp b/utils/dentry/src/file_utils.cpp index e7a73753d..823ad5626 100644 --- a/utils/dentry/src/file_utils.cpp +++ b/utils/dentry/src/file_utils.cpp @@ -31,8 +31,8 @@ int64_t FileUtils::ReadFile(int fd, off_t offset, size_t size, void *data) return -1; } - off_t ret = lseek(fd, offset, SEEK_SET); - if (ret < 0) { + off_t err = lseek(fd, offset, SEEK_SET); + if (err < 0) { LOGE("lseek failed, errno %{public}d, offset %{public}ld, fd=%{public}d", errno, static_cast(offset), fd); return -errno; } @@ -60,8 +60,8 @@ int64_t FileUtils::WriteFile(int fd, const void *data, off_t offset, size_t size return -1; } - off_t ret = lseek(fd, offset, SEEK_SET); - if (ret < 0) { + off_t err = lseek(fd, offset, SEEK_SET); + if (err < 0) { LOGE("lseek failed, errno %{public}d, offset %{public}d, fd=%{public}d", errno, static_cast(offset), fd); return -errno; } diff --git a/utils/dentry/src/meta_file.cpp b/utils/dentry/src/meta_file.cpp index 02d10a08d..501633d26 100644 --- a/utils/dentry/src/meta_file.cpp +++ b/utils/dentry/src/meta_file.cpp @@ -43,6 +43,7 @@ constexpr uint32_t BITS_PER_BYTE = 8; constexpr uint32_t HMDFS_SLOT_LEN_BITS = 3; constexpr uint64_t DELTA = 0x9E3779B9; /* Hashing code copied from f2fs */ constexpr uint64_t HMDFS_HASH_COL_BIT = (0x1ULL) << 63; +constexpr uint32_t DIR_SIZE = 4096; #pragma pack(push, 1) struct HmdfsDentry { @@ -80,10 +81,9 @@ static uint64_t PathHash(const std::string &path, bool caseSense) { uint64_t res = 0; const char *kp = path.c_str(); - char c; while (*kp) { - c = *kp; + char c = *kp; if (!caseSense) { c = tolower(c); } @@ -116,8 +116,47 @@ static std::string GetDentryfileByPath(uint32_t userId, const std::string &path, return cacheDir + dentryFileName; } +static std::string GetParentDir(const std::string &path) +{ + if ((path == "/") || (path == "")) { + return ""; + } + + auto pos = path.find_last_of('/'); + if ((pos == std::string::npos) || (pos == 0)) { + return "/"; + } + + return path.substr(0, pos); +} + +static std::string GetFileName(const std::string &path) +{ + if ((path == "/") || (path == "")) { + return ""; + } + + auto pos = path.find_last_of('/'); + if (pos == std::string::npos) { + return ""; + } + + return path.substr(pos + 1); +} + +static std::shared_ptr GetParentMetaFile(uint32_t userId, const std::string &path) +{ + std::string parentPath = GetParentDir(path); + if (parentPath == "") { + return nullptr; + } + + return MetaFileMgr::GetInstance().GetMetaFile(userId, parentPath); +} + MetaFile::MetaFile(uint32_t userId, const std::string &path) { + path_ = path; cacheFile_ = GetDentryfileByPath(userId, path); fd_ = UniqueFd{open(cacheFile_.c_str(), O_RDWR | O_CREAT)}; LOGD("fd=%{public}d, errno :%{public}d", fd_.Get(), errno); @@ -135,16 +174,36 @@ MetaFile::MetaFile(uint32_t userId, const std::string &path) (void)FileUtils::ReadFile(fd_, 0, sizeof(header), &header); dentryCount_ = header.dentryCount; - // GetParentMetaFile - // if parentMetaFile_->DoLookup() - // parentMetaFile_->DoCreate() + /* lookup and create in parent */ + parentMetaFile_ = GetParentMetaFile(userId, path); + std::string dirName = GetFileName(path); + if ((parentMetaFile_ == nullptr) || (dirName == "")) { + return; + } + MetaBase m(dirName, std::to_string(PathHash(path, false))); + ret = parentMetaFile_->DoLookup(m); + if (ret != E_OK) { + m.mode = S_IFDIR; + m.size = DIR_SIZE; + ret = parentMetaFile_->DoCreate(m); + if (ret != E_OK) { + LOGE("create parent failed, ret %{public}d", ret); + } + } } MetaFile::~MetaFile() { if ((dentryCount_ == 0) && (path_ != "/")) { unlink(cacheFile_.c_str()); - // parentMetaFile_->DoRemove() + if (parentMetaFile_) { + std::string dirName = GetFileName(path_); + if (dirName == "") { + return; + } + MetaBase m(dirName, std::to_string(PathHash(path_, false))); + parentMetaFile_->DoRemove(m); + } return; } @@ -615,5 +674,71 @@ int32_t MetaFile::DoRename(const MetaBase &oldBase, const std::string &newName) return E_OK; } + +static int32_t DecodeDentrys(const HmdfsDentryGroup &dentryGroup, std::vector &bases) +{ + for (int i = 0; i < DENTRY_PER_GROUP; i++) { + int len = dentryGroup.nsl[i].namelen; + if (!BitOps::TestBit(i, dentryGroup.bitmap) || len == 0 || len >= PATH_MAX) { + continue; + } + + std::string name(reinterpret_cast(dentryGroup.fileName[i]), len); + + MetaBase base(name); + base.mode = dentryGroup.nsl[i].mode; + base.mtime = dentryGroup.nsl[i].mtime; + base.size = dentryGroup.nsl[i].size; + base.cloudId = std::string(reinterpret_cast(dentryGroup.nsl[i].recordId), CLOUD_RECORD_ID_LEN); + bases.emplace_back(base); + } + return 0; +} + +int32_t MetaFile::LoadChildren(std::vector &bases) +{ + if (fd_ < 0) { + LOGE("bad metafile fd"); + return EINVAL; + } + + std::lock_guard lock(mtx_); + struct stat fileStat; + int ret = fstat(fd_, &fileStat); + if (ret != E_OK) { + return EINVAL; + } + + uint64_t fileSize = fileStat.st_size; + uint64_t groupCnt = GetDentryGroupCnt(fileSize); + HmdfsDentryGroup dentryGroup; + + for (int i = 1; i < groupCnt + 1; i++) { + uint64_t off = i * sizeof(HmdfsDentryGroup); + FileUtils::ReadFile(fd_, off, sizeof(HmdfsDentryGroup), &dentryGroup); + DecodeDentrys(dentryGroup, bases); + } + return E_OK; +} + +std::shared_ptr MetaFileMgr::GetMetaFile(uint32_t userId, const std::string &path) +{ + std::shared_ptr mFile = nullptr; + std::lock_guard lock(mtx_); + + if (metaFiles_.find({userId, path}) != metaFiles_.end()) { + mFile = metaFiles_[{userId, path}]; + } else { + mFile = std::make_shared(userId, path); + metaFiles_[{userId, path}] = mFile; + } + return mFile; +} + +void MetaFileMgr::ClearAll() +{ + metaFiles_.clear(); +} + } // namespace FileManagement } // namespace OHOS -- Gitee From c4ebebacedc356096e206d6faffa59410475ce16 Mon Sep 17 00:00:00 2001 From: stonexx Date: Fri, 5 May 2023 13:06:57 +0800 Subject: [PATCH 2/3] dentry: new !252 Signed-off-by: stonexx Change-Id: I038356ada3c1dfe725029358857d1a34b429e30f --- .../gallery_data_sync/gallery_file_const.h | 4 ++ .../gallery_data_sync/data_convertor.cpp | 45 ++++++++++++++ .../gallery_data_sync/file_data_handler.cpp | 62 ++++++++++++++++++- .../unittests/cloudsync_sa/data_sync/BUILD.gn | 1 + test/unittests/cloudsync_sa/ipc/BUILD.gn | 1 + utils/dentry/include/meta_file.h | 3 + utils/dentry/src/meta_file.cpp | 34 ++++++++++ 7 files changed, 147 insertions(+), 3 deletions(-) diff --git a/services/cloudsyncservice/include/data_sync/gallery_data_sync/gallery_file_const.h b/services/cloudsyncservice/include/data_sync/gallery_data_sync/gallery_file_const.h index a88718cbe..b3f5155d8 100644 --- a/services/cloudsyncservice/include/data_sync/gallery_data_sync/gallery_file_const.h +++ b/services/cloudsyncservice/include/data_sync/gallery_data_sync/gallery_file_const.h @@ -18,6 +18,10 @@ #include +#include "data_convertor.h" +#include "medialibrary_db_const.h" +#include "medialibrary_type_const.h" + namespace OHOS { namespace FileManagement { namespace CloudSync { diff --git a/services/cloudsyncservice/src/data_sync/gallery_data_sync/data_convertor.cpp b/services/cloudsyncservice/src/data_sync/gallery_data_sync/data_convertor.cpp index cd2de252e..a88b2ab36 100644 --- a/services/cloudsyncservice/src/data_sync/gallery_data_sync/data_convertor.cpp +++ b/services/cloudsyncservice/src/data_sync/gallery_data_sync/data_convertor.cpp @@ -18,6 +18,7 @@ #include "rdb_errno.h" #include "dfs_error.h" +#include "gallery_file_const.h" #include "utils_log.h" #include "sdk_helper.h" @@ -110,6 +111,50 @@ int32_t DataConvertor::GetString(const string &key, string &val, NativeRdb::Resu int32_t DataConvertor::RecordToValueBucket(const DriveKit::DKRecord &record, NativeRdb::ValuesBucket &valueBucket) { + DriveKit::DKRecordData data; + record.GetRecordData(data); + DriveKit::DKRecordFieldMap properties = data[FILE_PROPERTIES]; + + auto size = GALLERY_FILE_COLUMNS.size(); + for (int i = 0 ; i < size; i++) { + auto field = GALLERY_FILE_COLUMNS[i]; + auto type = GALLERY_FILE_COLUMN_TYPES[i]; + if (properties.find(field) == properties.end()) { + LOGE("filed %{public}s not found in record.properties", field.c_str()); + continue; + } + switch (type) { + case DataType::INT: { + int value = properties[field]; + valueBucket.PutInt(field, value); + break; + } + case DataType::LONG: { + int64_t value = properties[field]; + valueBucket.PutInt(field, value); + break; + } + case DataType::STRING: { + string value = properties[field]; + valueBucket.PutString(field, value); + break; + } + case DataType::BOOL: { + bool value = properties[field]; + valueBucket.PutInt(field, value); + break; + } + case DataType::DOUBLE: { + double value = properties[field]; + valueBucket.PutDouble(field, value); + break; + } + default: { + LOGE("invalid data type %{public}d", static_cast(type)); + break; + } + } + } return E_OK; } } // namespace CloudSync diff --git a/services/cloudsyncservice/src/data_sync/gallery_data_sync/file_data_handler.cpp b/services/cloudsyncservice/src/data_sync/gallery_data_sync/file_data_handler.cpp index 7579507bb..afafa0d69 100644 --- a/services/cloudsyncservice/src/data_sync/gallery_data_sync/file_data_handler.cpp +++ b/services/cloudsyncservice/src/data_sync/gallery_data_sync/file_data_handler.cpp @@ -18,6 +18,7 @@ #include "dfs_error.h" #include "utils_log.h" #include "gallery_file_const.h" +#include "meta_file.h" namespace OHOS { namespace FileManagement { @@ -56,6 +57,7 @@ void FileDataHandler::GetFetchCondition(FetchCondition &cond) int32_t FileDataHandler::OnFetchRecords(const shared_ptr> &records) { + LOGI("on fetch %{public}zu records", records->size()); int32_t ret = E_OK; for (const auto &it : *records) { const auto &record = it; @@ -102,18 +104,72 @@ int32_t FileDataHandler::OnFetchRecords(const shared_ptr> &reco break; } } + MetaFileMgr::GetInstance().ClearAll(); return ret; } +static int32_t DentryInsert(int userId, const DKRecord &record) +{ + DKRecordData data; + record.GetRecordData(data); + if (data.find(FILE_PROPERTIES) == data.end()) { + LOGE("record data cannot find properties"); + return E_INVAL_ARG; + } + DriveKit::DKRecordFieldMap prop = data[FILE_PROPERTIES]; + if (prop.find(MEDIA_DATA_DB_FILE_PATH) == prop.end() || prop.find(MEDIA_DATA_DB_SIZE) == prop.end() || + prop.find(MEDIA_DATA_DB_DATE_MODIFIED) == prop.end()) { + LOGE("record data cannot find some properties"); + return E_INVAL_ARG; + } + + const string sandboxPrefix = "/storage/media/local"; + string fullPath = prop[MEDIA_DATA_DB_FILE_PATH]; + size_t pos = fullPath.find_first_of(sandboxPrefix); + size_t lpos = fullPath.find_last_of("/"); + if (pos != 0 || pos == string::npos || lpos == string::npos) { + LOGE("invalid path %{private}s", fullPath.c_str()); + return E_INVAL_ARG; + } + string relativePath = fullPath.substr(sandboxPrefix.length(), lpos - sandboxPrefix.length()); + relativePath = (relativePath == "") ? "/" : relativePath; + string fileName = fullPath.substr(lpos + 1); + + string rawRecordId = record.GetRecordId(); + string cloudId = MetaFileMgr::RecordIdToCloudId(rawRecordId); + int64_t isize = prop[MEDIA_DATA_DB_SIZE]; + int64_t mtime = prop[MEDIA_DATA_DB_DATE_MODIFIED]; + + auto mFile = MetaFileMgr::GetInstance().GetMetaFile(userId, relativePath); + MetaBase mBaseLookup(fileName); + MetaBase mBase(fileName, cloudId); + mBase.size = isize; + mBase.mtime = mtime; + if (mFile->DoLookup(mBaseLookup) == E_OK) { + LOGE("dentry exist when insert, do update instead"); + return mFile->DoUpdate(mBase); + } + return mFile->DoCreate(mBase); +} + int32_t FileDataHandler::PullRecordInsert(const DKRecord &record) { - /* insert hmdfs dentry file here */ + /* check local file conflict */ + int ret = DentryInsert(userId_, record); + if (ret != E_OK) { + LOGE("MetaFile Create failed %{public}d", ret); + return ret; + } int64_t rowId; ValuesBucket values; - createConvertor_.RecordToValueBucket(record, values); + ret = createConvertor_.RecordToValueBucket(record, values); + if (ret != E_OK) { + LOGE("record to valuebucket failed"); + return E_INVAL_ARG; + } - int ret = Insert(rowId, values); + ret = Insert(rowId, values); if (ret != E_OK) { LOGE("Insert pull record failed"); return E_RDB; diff --git a/test/unittests/cloudsync_sa/data_sync/BUILD.gn b/test/unittests/cloudsync_sa/data_sync/BUILD.gn index 6fb083488..11a4ccad6 100644 --- a/test/unittests/cloudsync_sa/data_sync/BUILD.gn +++ b/test/unittests/cloudsync_sa/data_sync/BUILD.gn @@ -45,6 +45,7 @@ ohos_unittest("data_sync_manager_test") { ] deps = [ + "${utils_path}:libdistributedfiledentry", "${utils_path}:libdistributedfileutils", "//third_party/googletest:gmock_main", "//third_party/googletest:gtest_main", diff --git a/test/unittests/cloudsync_sa/ipc/BUILD.gn b/test/unittests/cloudsync_sa/ipc/BUILD.gn index 2ccb4a07a..11f7b2217 100644 --- a/test/unittests/cloudsync_sa/ipc/BUILD.gn +++ b/test/unittests/cloudsync_sa/ipc/BUILD.gn @@ -81,6 +81,7 @@ ohos_unittest("cloud_sync_service_test") { ] deps = [ + "${utils_path}:libdistributedfiledentry", "${utils_path}:libdistributedfileutils", "//third_party/googletest:gmock_main", "//third_party/googletest:gtest_main", diff --git a/utils/dentry/include/meta_file.h b/utils/dentry/include/meta_file.h index 47f8dbf74..a38a75646 100644 --- a/utils/dentry/include/meta_file.h +++ b/utils/dentry/include/meta_file.h @@ -58,6 +58,9 @@ public: static MetaFileMgr instance_; return instance_; } + /* recordId is hex string of 256 bits, convert to u8 cloudId[32] to kernel */ + static std::string RecordIdToCloudId(const std::string hexStr); + static std::string CloudIdToRecordId(const std::string cloudId); std::shared_ptr GetMetaFile(uint32_t userId, const std::string &path); void ClearAll(); private: diff --git a/utils/dentry/src/meta_file.cpp b/utils/dentry/src/meta_file.cpp index 501633d26..825ecb11d 100644 --- a/utils/dentry/src/meta_file.cpp +++ b/utils/dentry/src/meta_file.cpp @@ -16,6 +16,7 @@ #include "meta_file.h" #include +#include #include #include @@ -740,5 +741,38 @@ void MetaFileMgr::ClearAll() metaFiles_.clear(); } +std::string MetaFileMgr::RecordIdToCloudId(const std::string hexStr) +{ + std::string result; + + for (std::size_t i = 0; i < hexStr.length(); i += 2) { + std::string hexByte = hexStr.substr(i, 2); + char *endPtr; + unsigned long hexValue = std::strtoul(hexByte.c_str(), &endPtr, 16); + + if (endPtr != hexByte.c_str() + hexByte.length()) { + LOGE("Invalid hexadecimal string: %{public}s", hexStr.c_str()); + return ""; + } + result += static_cast(hexValue); + } + if (result.size() > CLOUD_RECORD_ID_LEN) { + LOGE("Invalid result length %{public}zu", result.size()); + return ""; + } + + return result; +} + +std::string MetaFileMgr::CloudIdToRecordId(const std::string cloudId) +{ + std::stringstream result; + for (std::size_t i = 0; i < cloudId.length(); i++) { + uint8_t u8Byte = cloudId[i]; + result << std::setw(2) << std::setfill('0') << std::hex << static_cast(u8Byte); + } + return result.str(); +} + } // namespace FileManagement } // namespace OHOS -- Gitee From eb3ef19dbe3e9baa3331e3fae47fbb59504a5a9f Mon Sep 17 00:00:00 2001 From: stonexx Date: Fri, 5 May 2023 14:31:10 +0800 Subject: [PATCH 3/3] dentry: fix compile error Signed-off-by: stonexx Change-Id: If02b684a5a1f90545566440105c561755e6d2453 --- .../include/data_sync/gallery_data_sync/gallery_file_const.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/cloudsyncservice/include/data_sync/gallery_data_sync/gallery_file_const.h b/services/cloudsyncservice/include/data_sync/gallery_data_sync/gallery_file_const.h index 5eccba508..6b204edcd 100644 --- a/services/cloudsyncservice/include/data_sync/gallery_data_sync/gallery_file_const.h +++ b/services/cloudsyncservice/include/data_sync/gallery_data_sync/gallery_file_const.h @@ -19,6 +19,8 @@ #include #include "data_convertor.h" +#include "medialibrary_db_const.h" +#include "medialibrary_type_const.h" namespace OHOS { namespace FileManagement { -- Gitee