diff --git a/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/LICENSE b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/README.md b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4270fc7b7128ea55a4a3384cecf5842e0bde083c --- /dev/null +++ b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/README.md @@ -0,0 +1,43 @@ +# RecVAE +The official PyTorch implementation of the paper "RecVAE: A New Variational Autoencoder for Top-N Recommendations with Implicit Feedback". + +In order to train RecVAE on MovieLens-20M dataset ([download link](http://files.grouplens.org/datasets/movielens/ml-20m.zip)), preprocess it using following script: + +```sh +python preprocessing.py --dataset --output_dir --threshold 3.5 --heldout_users 10000 +``` + +You can also use another dataset, it should contain columns `userId`, `movieId` and `rating` (in arbitrary order). + +Then use the following command to start training: + +``` +python run.py --dataset +``` + +See `python run.py -h` for more information. + +Current model is slightly different from the one described in the paper, you can find original code [here](https://github.com/ilya-shenbin/RecVAE/tree/wsdm). + +Some sources from [Variational autoencoders for collaborative filtering](https://github.com/dawenl/vae_cf) is partially used. + +If you used this code for a publication, please cite our WSDM'20 paper +``` +@inproceedings{10.1145/3336191.3371831, + author = {Shenbin, Ilya and Alekseev, Anton and Tutubalina, Elena and Malykh, Valentin and Nikolenko, Sergey I.}, + title = {RecVAE: A New Variational Autoencoder for Top-N Recommendations with Implicit Feedback}, + year = {2020}, + isbn = {9781450368223}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + url = {https://doi.org/10.1145/3336191.3371831}, + doi = {10.1145/3336191.3371831}, + booktitle = {Proceedings of the 13th International Conference on Web Search and Data Mining}, + pages = {528–536}, + numpages = {9}, + keywords = {deep learning, collaborative filtering, variational autoencoders}, + location = {Houston, TX, USA}, + series = {WSDM ’20} +} +``` + diff --git a/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/model.py b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/model.py new file mode 100644 index 0000000000000000000000000000000000000000..1b2b776b0f13e4db1878c99eb114e1c5331a1360 --- /dev/null +++ b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/model.py @@ -0,0 +1,118 @@ +import numpy as np +from copy import deepcopy + +import torch +from torch import nn +from torch.nn import functional as F + + +def swish(x): + return x.mul(torch.sigmoid(x)) + +def log_norm_pdf(x, mu, logvar): + return -0.5*(logvar + np.log(2 * np.pi) + (x - mu).pow(2) / logvar.exp()) + + +class CompositePrior(nn.Module): + def __init__(self, hidden_dim, latent_dim, input_dim, mixture_weights=[3/20, 3/4, 1/10]): + super(CompositePrior, self).__init__() + + self.mixture_weights = mixture_weights + + self.mu_prior = nn.Parameter(torch.Tensor(1, latent_dim), requires_grad=False) + self.mu_prior.data.fill_(0) + + self.logvar_prior = nn.Parameter(torch.Tensor(1, latent_dim), requires_grad=False) + self.logvar_prior.data.fill_(0) + + self.logvar_uniform_prior = nn.Parameter(torch.Tensor(1, latent_dim), requires_grad=False) + self.logvar_uniform_prior.data.fill_(10) + + self.encoder_old = Encoder(hidden_dim, latent_dim, input_dim) + self.encoder_old.requires_grad_(False) + + def forward(self, x, z): + post_mu, post_logvar = self.encoder_old(x, 0) + + stnd_prior = log_norm_pdf(z, self.mu_prior, self.logvar_prior) + post_prior = log_norm_pdf(z, post_mu, post_logvar) + unif_prior = log_norm_pdf(z, self.mu_prior, self.logvar_uniform_prior) + + gaussians = [stnd_prior, post_prior, unif_prior] + gaussians = [g.add(np.log(w)) for g, w in zip(gaussians, self.mixture_weights)] + + density_per_gaussian = torch.stack(gaussians, dim=-1) + + return torch.logsumexp(density_per_gaussian, dim=-1) + + +class Encoder(nn.Module): + def __init__(self, hidden_dim, latent_dim, input_dim, eps=1e-1): + super(Encoder, self).__init__() + + self.fc1 = nn.Linear(input_dim, hidden_dim) + self.ln1 = nn.LayerNorm(hidden_dim, eps=eps) + self.fc2 = nn.Linear(hidden_dim, hidden_dim) + self.ln2 = nn.LayerNorm(hidden_dim, eps=eps) + self.fc3 = nn.Linear(hidden_dim, hidden_dim) + self.ln3 = nn.LayerNorm(hidden_dim, eps=eps) + self.fc4 = nn.Linear(hidden_dim, hidden_dim) + self.ln4 = nn.LayerNorm(hidden_dim, eps=eps) + self.fc5 = nn.Linear(hidden_dim, hidden_dim) + self.ln5 = nn.LayerNorm(hidden_dim, eps=eps) + self.fc_mu = nn.Linear(hidden_dim, latent_dim) + self.fc_logvar = nn.Linear(hidden_dim, latent_dim) + + def forward(self, x, dropout_rate): + norm = x.pow(2).sum(dim=-1).sqrt() + x = x / norm[:, None] + + x = F.dropout(x, p=dropout_rate, training=self.training) + + h1 = self.ln1(swish(self.fc1(x))) + h2 = self.ln2(swish(self.fc2(h1) + h1)) + h3 = self.ln3(swish(self.fc3(h2) + h1 + h2)) + h4 = self.ln4(swish(self.fc4(h3) + h1 + h2 + h3)) + h5 = self.ln5(swish(self.fc5(h4) + h1 + h2 + h3 + h4)) + return self.fc_mu(h5), self.fc_logvar(h5) + + +class VAE(nn.Module): + def __init__(self, hidden_dim, latent_dim, input_dim): + super(VAE, self).__init__() + + self.encoder = Encoder(hidden_dim, latent_dim, input_dim) + self.prior = CompositePrior(hidden_dim, latent_dim, input_dim) + self.decoder = nn.Linear(latent_dim, input_dim) + + def reparameterize(self, mu, logvar): + if self.training: + std = torch.exp(0.5*logvar) + eps = torch.randn_like(std) + return eps.mul(std).add_(mu) + else: + return mu + + def forward(self, user_ratings, beta=None, gamma=1, dropout_rate=0.5, calculate_loss=True): + mu, logvar = self.encoder(user_ratings, dropout_rate=dropout_rate) + z = self.reparameterize(mu, logvar) + x_pred = self.decoder(z) + + if calculate_loss: + if gamma: + norm = user_ratings.sum(dim=-1) + kl_weight = gamma * norm + elif beta: + kl_weight = beta + + mll = (F.log_softmax(x_pred, dim=-1) * user_ratings).sum(dim=-1).mean() + kld = (log_norm_pdf(z, mu, logvar) - self.prior(user_ratings, z)).sum(dim=-1).mul(kl_weight).mean() + negative_elbo = -(mll - kld) + + return (mll, kld), negative_elbo + + else: + return x_pred + + def update_prior(self): + self.prior.encoder_old.load_state_dict(deepcopy(self.encoder.state_dict())) diff --git a/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/modelzoo_level.txt b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/modelzoo_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b49b4fb26c2694a86567bea1b462e7dcb03cc31 --- /dev/null +++ b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/modelzoo_level.txt @@ -0,0 +1,3 @@ +FuncStatus:OK +PerfStatus:OK +PrecisionStatus:OK \ No newline at end of file diff --git a/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/preprocessing.py b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/preprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..808da2960128d0828f921a648e6a8802c3aa00f6 --- /dev/null +++ b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/preprocessing.py @@ -0,0 +1,149 @@ +# based on https://github.com/dawenl/vae_cf + +import os +import sys + +import numpy as np +from scipy import sparse +import pandas as pd + +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('--dataset', type=str) +parser.add_argument('--output_dir', type=str) +parser.add_argument('--threshold', type=float) +parser.add_argument('--min_items_per_user', type=int, default=5) +parser.add_argument('--min_users_per_item', type=int, default=0) +parser.add_argument('--heldout_users', type=int) + +args = parser.parse_args() + +dataset = args.dataset +output_dir = args.output_dir +threshold = args.threshold +min_uc = args.min_items_per_user +min_sc = args.min_users_per_item +n_heldout_users = args.heldout_users + +raw_data = pd.read_csv(dataset, header=0) +raw_data = raw_data[raw_data['rating'] > threshold] +raw_data.head() + + +def get_count(tp, id): + playcount_groupbyid = tp[[id]].groupby(id, as_index=False) + count = playcount_groupbyid.size() + return count + + +def filter_triplets(tp, min_uc=min_uc, min_sc=min_sc): + if min_sc > 0: + itemcount = get_count(tp, 'movieId') + tp = tp[tp['movieId'].isin(itemcount.index[itemcount >= min_sc])] + + if min_uc > 0: + usercount = get_count(tp, 'userId') + tp = tp[tp['userId'].isin(usercount.index[usercount >= min_uc])] + + usercount, itemcount = get_count(tp, 'userId'), get_count(tp, 'movieId') + return tp, usercount, itemcount + + +raw_data, user_activity, item_popularity = filter_triplets(raw_data) + +sparsity = 1. * raw_data.shape[0] / (user_activity.shape[0] * item_popularity.shape[0]) + +print("After filtering, there are %d watching events from %d users and %d movies (sparsity: %.3f%%)" % + (raw_data.shape[0], user_activity.shape[0], item_popularity.shape[0], sparsity * 100)) + +unique_uid = user_activity.index + +np.random.seed(98765) +idx_perm = np.random.permutation(unique_uid.size) +unique_uid = unique_uid[idx_perm] + +n_users = unique_uid.size + +tr_users = unique_uid[:(n_users - n_heldout_users * 2)] +vd_users = unique_uid[(n_users - n_heldout_users * 2): (n_users - n_heldout_users)] +te_users = unique_uid[(n_users - n_heldout_users):] + +train_plays = raw_data.loc[raw_data['userId'].isin(tr_users)] + +unique_sid = pd.unique(train_plays['movieId']) + +show2id = dict((sid, i) for (i, sid) in enumerate(unique_sid)) +profile2id = dict((pid, i) for (i, pid) in enumerate(unique_uid)) + +if not os.path.exists(output_dir): + os.makedirs(output_dir) + +with open(os.path.join(output_dir, 'unique_sid.txt'), 'w') as f: + for sid in unique_sid: + f.write('%s\n' % sid) + +with open(os.path.join(output_dir, 'unique_uid.txt'), 'w') as f: + for uid in unique_uid: + f.write('%s\n' % uid) + + +def split_train_test_proportion(data, test_prop=0.2): + data_grouped_by_user = data.groupby('userId') + tr_list, te_list = list(), list() + + np.random.seed(98765) + + for i, (_, group) in enumerate(data_grouped_by_user): + n_items_u = len(group) + + if n_items_u >= 5: + idx = np.zeros(n_items_u, dtype='bool') + idx[np.random.choice(n_items_u, size=int(test_prop * n_items_u), replace=False).astype('int64')] = True + + tr_list.append(group[np.logical_not(idx)]) + te_list.append(group[idx]) + else: + tr_list.append(group) + + if i % 1000 == 0: + print("%d users sampled" % i) + sys.stdout.flush() + + data_tr = pd.concat(tr_list) + data_te = pd.concat(te_list) + + return data_tr, data_te + + +vad_plays = raw_data.loc[raw_data['userId'].isin(vd_users)] +vad_plays = vad_plays.loc[vad_plays['movieId'].isin(unique_sid)] + +vad_plays_tr, vad_plays_te = split_train_test_proportion(vad_plays) + +test_plays = raw_data.loc[raw_data['userId'].isin(te_users)] +test_plays = test_plays.loc[test_plays['movieId'].isin(unique_sid)] + +test_plays_tr, test_plays_te = split_train_test_proportion(test_plays) + +def numerize(tp): + uid = list(map(lambda x: profile2id[x], tp['userId'])) + sid = list(map(lambda x: show2id[x], tp['movieId'])) + return pd.DataFrame(data={'uid': uid, 'sid': sid}, columns=['uid', 'sid']) + + +train_data = numerize(train_plays) +train_data.to_csv(os.path.join(output_dir, 'train.csv'), index=False) + +vad_data_tr = numerize(vad_plays_tr) +vad_data_tr.to_csv(os.path.join(output_dir, 'validation_tr.csv'), index=False) + +vad_data_te = numerize(vad_plays_te) +vad_data_te.to_csv(os.path.join(output_dir, 'validation_te.csv'), index=False) + +test_data_tr = numerize(test_plays_tr) +test_data_tr.to_csv(os.path.join(output_dir, 'test_tr.csv'), index=False) + +test_data_te = numerize(test_plays_te) +test_data_te.to_csv(os.path.join(output_dir, 'test_te.csv'), index=False) + diff --git a/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/run.py b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/run.py new file mode 100644 index 0000000000000000000000000000000000000000..f95769198cf228c8622c4d7ba9c9721a05a7df9a --- /dev/null +++ b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/run.py @@ -0,0 +1,230 @@ +import numpy as np + +import torch +from torch import optim + +import random +from copy import deepcopy + +from utils import get_data, ndcg, recall +from model import VAE +import time +import apex +from apex import amp + +import argparse +parser = argparse.ArgumentParser() +parser.add_argument('--dataset', type=str) +parser.add_argument('--hidden-dim', type=int, default=600) +parser.add_argument('--latent-dim', type=int, default=200) +parser.add_argument('--batch-size', type=int, default=500) +parser.add_argument('--beta', type=float, default=None) +parser.add_argument('--gamma', type=float, default=0.005) +parser.add_argument('--lr', type=float, default=5e-4) +parser.add_argument('--n-epochs', type=int, default=50) +parser.add_argument('--n-enc_epochs', type=int, default=3) +parser.add_argument('--n-dec_epochs', type=int, default=1) +parser.add_argument('--not-alternating', type=bool, default=False) +# Mixed precision training parameters +parser.add_argument('--apex', action='store_true', + help='Use apex for mixed precision training') +parser.add_argument('--apex-opt-level', default='O1', type=str, + help='For apex mixed precision training' + 'O0 for FP32 training, O1 for mixed precision training.' + 'For further detail, see https://github.com/NVIDIA/apex/tree/master/examples/imagenet') +parser.add_argument('--loss-scale-value', default=1024., type=float, + help='loss scale using in amp, default -1 means dynamic') +## for ascend 910 +parser.add_argument('--device_id', default=5, type=int, help='device id') +args = parser.parse_args() + +seed = 1337 +random.seed(seed) +np.random.seed(seed) +torch.manual_seed(seed) + +#device = torch.device("npu") +device = torch.device(f'npu:{args.device_id}') +torch.npu.set_device(device) +print("Use NPU: {} for training".format(args.device_id)) + +data = get_data(args.dataset) +train_data, valid_in_data, valid_out_data, test_in_data, test_out_data = data + + +def generate(batch_size, device, data_in, data_out=None, shuffle=False, samples_perc_per_epoch=1): + assert 0 < samples_perc_per_epoch <= 1 + + total_samples = data_in.shape[0] + samples_per_epoch = int(total_samples * samples_perc_per_epoch) + + if shuffle: + idxlist = np.arange(total_samples) + np.random.shuffle(idxlist) + idxlist = idxlist[:samples_per_epoch] + else: + idxlist = np.arange(samples_per_epoch) + + for st_idx in range(0, samples_per_epoch, batch_size): + end_idx = min(st_idx + batch_size, samples_per_epoch) + idx = idxlist[st_idx:end_idx] + + yield Batch(device, idx, data_in, data_out) + + +class Batch: + def __init__(self, device, idx, data_in, data_out=None): + self._device = device + self._idx = idx + self._data_in = data_in + self._data_out = data_out + + def get_idx(self): + return self._idx + + def get_idx_to_dev(self): + return torch.LongTensor(self.get_idx()).to(self._device) + + def get_ratings(self, is_out=False): + data = self._data_out if is_out else self._data_in + return data[self._idx] + + def get_ratings_to_dev(self, is_out=False): + return torch.Tensor( + self.get_ratings(is_out).toarray() + ).to(self._device,non_blocking=True) + + +def evaluate(model, data_in, data_out, metrics, samples_perc_per_epoch=1, batch_size=500): + metrics = deepcopy(metrics) + model.eval() + + for m in metrics: + m['score'] = [] + + for batch in generate(batch_size=batch_size, + device=device, + data_in=data_in, + data_out=data_out, + samples_perc_per_epoch=samples_perc_per_epoch + ): + + ratings_in = batch.get_ratings_to_dev() + ratings_out = batch.get_ratings(is_out=True) + + ratings_pred = model(ratings_in, calculate_loss=False).cpu().detach().numpy()\ + #ratings_pred = model.forward(ratings_in, calculate_loss=False).cpu().detach().numpy() + + if not (data_in is data_out): + ratings_pred[batch.get_ratings().nonzero()] = -np.inf + + for m in metrics: + m['score'].append(m['metric'](ratings_pred, ratings_out, k=m['k'])) + + for m in metrics: + m['score'] = np.concatenate(m['score']).mean() + + return [x['score'] for x in metrics] + + +def run(model, opts, train_data, batch_size, n_epochs, beta, gamma, dropout_rate): + model.train() + step=0 + import time + #if args.apex: + # model, opts = amp.initialize(model, opts, + # opt_level=args.apex_opt_level, + # loss_scale=args.loss_scale_value, + # combine_grad=True) + for epoch in range(n_epochs): + for batch in generate(batch_size=batch_size, device=device, data_in=train_data, shuffle=True): + ratings = batch.get_ratings_to_dev() + start_time = time.time() + for optimizer in opts: + optimizer.zero_grad() + step+=1 + _, loss = model(ratings, beta=beta, gamma=gamma, dropout_rate=dropout_rate) + if args.apex: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + for optimizer in opts: + optimizer.step() + #loss.backward() + print('epoch: {}, step:{} loss: {:.2f}, time: {:.3f}'.format(epoch, step, loss, time.time() - start_time)) + for optimizer in opts: + optimizer.step() + + +model_kwargs = { + 'hidden_dim': args.hidden_dim, + 'latent_dim': args.latent_dim, + 'input_dim': train_data.shape[1] +} +metrics = [{'metric': ndcg, 'k': 100}] + +best_ndcg = -np.inf +train_scores, valid_scores = [], [] + +model = VAE(**model_kwargs).to(device) +model = model.to(device) +model_best = VAE(**model_kwargs).to(device) + +learning_kwargs = { + 'model': model, + 'train_data': train_data, + 'batch_size': args.batch_size, + 'beta': args.beta, + 'gamma': args.gamma +} + +decoder_params = set(model.decoder.parameters()) +encoder_params = set(model.encoder.parameters()) + +optimizer_encoder =apex.optimizers.NpuFusedAdam(encoder_params, lr=args.lr) +optimizer_decoder =apex.optimizers.NpuFusedAdam(decoder_params, lr=args.lr) + +if args.apex: + [model.decoder, model.encoder], [optimizer_decoder, optimizer_encoder] = amp.initialize([model.decoder, model.encoder], [optimizer_decoder, optimizer_encoder], + opt_level=args.apex_opt_level, + loss_scale=args.loss_scale_value, + combine_grad=True) + +for epoch in range(args.n_epochs): + #starttime=time.time() + #steptime=time.time() + + if args.not_alternating: + run(opts=[optimizer_encoder, optimizer_decoder], n_epochs=1, dropout_rate=0.5, **learning_kwargs) + else: + run(opts=[optimizer_encoder], n_epochs=args.n_enc_epochs, dropout_rate=0.5, **learning_kwargs) + model.update_prior() + run(opts=[optimizer_decoder], n_epochs=args.n_dec_epochs, dropout_rate=0, **learning_kwargs) + #print("FPS:",time.time()-steptime) + + train_scores.append( + evaluate(model, train_data, train_data, metrics, 0.01)[0] + ) + valid_scores.append( + evaluate(model, valid_in_data, valid_out_data, metrics, 1)[0] + ) + + if valid_scores[-1] > best_ndcg: + best_ndcg = valid_scores[-1] + model_best.load_state_dict(deepcopy(model.state_dict())) + + + print(f'epoch {epoch} | valid ndcg@100: {valid_scores[-1]:.4f} | ' + + f'best valid: {best_ndcg:.4f} | train ndcg@100: {train_scores[-1]:.4f}') + #print("Training:",time.time()-starttime) + + + + +test_metrics = [{'metric': ndcg, 'k': 100}, {'metric': recall, 'k': 20}, {'metric': recall, 'k': 50}] + +final_scores = evaluate(model_best, test_in_data, test_out_data, test_metrics) + +for metric, score in zip(test_metrics, final_scores): + print(f"{metric['metric'].__name__}@{metric['k']}:\t{score:.4f}") diff --git a/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/test/env.sh b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/test/env.sh new file mode 100644 index 0000000000000000000000000000000000000000..b46f2d4cf6e52d46424f4bbf79c6989fca8fe047 --- /dev/null +++ b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/test/env.sh @@ -0,0 +1,13 @@ +#!/bin/bash +cur_path=`pwd`/../ +export install_path=/usr/local/Ascend +export LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver:$LD_LIBRARY_PATH # 仅容器训练场景配置 +export PATH=${install_path}/fwkacllib/ccec_compiler/bin:${install_path}/fwkacllib/bin:$PATH +export LD_LIBRARY_PATH=${install_path}/fwkacllib/lib64:$LD_LIBRARY_PATH +export PYTHONPATH=${install_path}/fwkacllib/python/site-packages:$PYTHONPATH +export PYTHONPATH=/usr/local/python3.7.5/lib/python3.7/site-packages:${install_path}/tfplugin/python/site-packages:$PYTHONPATH +export ASCEND_OPP_PATH=${install_path}/opp +export ASCEND_AICPU_PATH=${install_path} +export JOB_ID=10087 +export ASCEND_GLOBAL_LOG_LEVEL=3 +export ASCEND_DEVICE_ID=0 diff --git a/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/test/train_full_1p.sh b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/test/train_full_1p.sh new file mode 100644 index 0000000000000000000000000000000000000000..143a639be0c16bc2d47da3b95856ba201da42e6d --- /dev/null +++ b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/test/train_full_1p.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +#当前路径,不需要修改 +cur_path=`pwd`/../ +path=`pwd` + +#集合通信参数,不需要修改 + +export RANK_SIZE=1 +export JOB_ID=10087 +RANK_ID_START=0 + +# 数据集路径,保持为空,不需要修改 +data_path="" + +#设置默认日志级别,不需要修改 +export ASCEND_GLOBAL_LOG_LEVEL=3 +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export NPU_CALCULATE_DEVICE=$ASCEND_DEVICE_ID + +#基础参数，需要模型审视修改 +#网络名称，同目录名称 +Network="RecVAE_ID0347_for_PyTorch" +#训练epoch +train_epochs=50 +#训练batch_size +batch_size=500 +#训练step +train_steps=1 +#学习率 +learning_rate=8e-2 +RANK_SIZE=1 + +#维测参数，precision_mode需要模型审视修改 +precision_mode="allow_mix_precision" +#维持参数，以下不需要修改 +over_dump=False +data_dump_flag=False +data_dump_step="10" +profiling=False +autotune=False +PREC="--apex --apex-opt-level O1" +# 帮助信息，不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_performance_1p.sh " + echo " " + echo "parameter explain: + --precision_mode precision mode(allow_fp32_to_fp16/force_fp16/must_keep_origin_dtype/allow_mix_precision) + --over_dump if or not over detection, default is False + --data_dump_flag data dump flag, default is False + --data_dump_step data dump step, default is 10 + --profiling if or not profiling for performance debug, default is False + --data_path source data of training + -h/--help show help message + " + exit 1 +fi + +#参数校验，不需要修改 +for para in $* +do + if [[ $para == --precision_mode* ]];then + apex_opt_level=`echo ${para#*=}` + if [[ $apex_opt_level != "O1" ]] && [[ $apex_opt_level != "O2" ]] && [[ $apex_opt_level != "O3" ]]; then + echo "[ERROR] para \"precision_mode\" must be config O1 or O2 or O3" + exit 1 + fi + PREC="--apex --apex-opt-level "$apex_opt_level + +elif [[ $para == --over_dump* ]];then + over_dump=`echo ${para#*=}` + over_dump_path=${cur_path}/output/overflow_dump + mkdir -p ${over_dump_path} + elif [[ $para == --data_dump_flag* ]];then + data_dump_flag=`echo ${para#*=}` + data_dump_path=${cur_path}/output/data_dump + mkdir -p ${data_dump_path} + elif [[ $para == --data_dump_step* ]];then + data_dump_step=`echo ${para#*=}` + elif [[ $para == --profiling* ]];then + profiling=`echo ${para#*=}` + profiling_dump_path=${cur_path}/output/profiling + mkdir -p ${profiling_dump_path} + elif [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + fi +done + +#校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be confing" + exit 1 +fi + +#进入训练脚本目录，需要模型审视修改 +cd $cur_path +#创建DeviceID输出目录，不需要修改 +if [ -d ${cur_path}/test/output/${ASCEND_DEVICE_ID} ];then + rm -rf ${cur_path}/test/output/${ASCEND_DEVICE_ID} + mkdir -p ${cur_path}/test/output/$ASCEND_DEVICE_ID/ckpt +else + mkdir -p ${cur_path}/test/output/$ASCEND_DEVICE_ID/ckpt +fi + + +#训练开始时间，不需要修改 +start_time=$(date +%s) +nohup python3 run.py --n-epochs=${train_epochs} --dataset=${data_path} ${PREC} \ + --device_id ${ASCEND_DEVICE_ID} > $cur_path/test/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log 2>&1 & +wait +#训练结束时间，不需要修改 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + + +#结果打印，不需要修改 +echo "------------------ Final result ------------------" +#输出性能FPS，需要模型审视修改 +TrainingTime=`grep "epoch" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |grep -wvE "step:234|step:1|step:2"|awk -F "time: " '{print $2}' |tail -n +3|awk '{sum+=$1} END {print"",sum/NR}'|sed s/[[:space:]]//g` + +FPS=`python3 -c "print(${batch_size}/${TrainingTime})"` +#打印，不需要修改 +echo "Final Performance images/sec : $FPS" + +#打印，不需要修改 +#echo "Final Train Accuracy : ${train_accuracy}" +echo "E2E Training Duration sec : $e2e_time" +#获取精度： +train_accuracy=`grep "best valid" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |awk -F "best valid: " '{print $2}'|awk -F " |" '{print $1}'|awk 'NR==1{max=$1;next}{max=max>$1?max:$1}END{print max}'` +#性能看护结果汇总 +#训练用例信息，不需要修改 +BatchSize=${batch_size} +DeviceType=`uname -m` +CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'acc' + +##获取性能数据，不需要修改 +#吞吐量 +ActualFPS=${FPS} +#单迭代训练时长 +#TrainingTime=`expr ${batch_size} \* 1000 / ${FPS}` + +#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中，需要根据模型审视 +grep "loss: " $path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log | awk -F "loss: " '{print $2}'|awk -F ", time" '{print $1}' > $path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt + +#最后一个迭代loss值，不需要修改 +ActualLoss=`awk 'END {print}' $path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt` + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${ActualFPS}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainAccuracy = ${train_accuracy}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log + diff --git a/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/test/train_performance_1p.sh b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/test/train_performance_1p.sh new file mode 100644 index 0000000000000000000000000000000000000000..3d683b95b91c9654ceb7e5cdce62a46393b4409b --- /dev/null +++ b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/test/train_performance_1p.sh @@ -0,0 +1,160 @@ +#!/bin/bash + +#当前路径,不需要修改 +cur_path=`pwd`/../ +path=`pwd` + +#集合通信参数,不需要修改 + +export RANK_SIZE=1 +export JOB_ID=10087 +RANK_ID_START=0 + +# 数据集路径,保持为空,不需要修改 +data_path="" + +#设置默认日志级别,不需要修改 +export ASCEND_GLOBAL_LOG_LEVEL=3 +export ASCEND_SLOG_PRINT_TO_STDOUT=0 + +#基础参数，需要模型审视修改 +#网络名称，同目录名称 +Network="RecVAE_ID0347_for_PyTorch" +#训练epoch +train_epochs=1 +#训练batch_size +batch_size=500 +#训练step +train_steps=1 +#学习率 +learning_rate=8e-2 +RANK_SIZE=1 + + +#维测参数，precision_mode需要模型审视修改 +precision_mode="allow_mix_precision" +#维持参数，以下不需要修改 +over_dump=False +data_dump_flag=False +data_dump_step="10" +profiling=False +autotune=False +PREC="--apex --apex-opt-level O1" +# 帮助信息，不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_performance_1p.sh " + echo " " + echo "parameter explain: + --precision_mode precision mode(allow_fp32_to_fp16/force_fp16/must_keep_origin_dtype/allow_mix_precision) + --over_dump if or not over detection, default is False + --data_dump_flag data dump flag, default is False + --data_dump_step data dump step, default is 10 + --profiling if or not profiling for performance debug, default is False + --data_path source data of training + -h/--help show help message + " + exit 1 +fi + +#参数校验，不需要修改 +for para in $* +do + if [[ $para == --precision_mode* ]];then + apex_opt_level=`echo ${para#*=}` + if [[ $apex_opt_level != "O1" ]] && [[ $apex_opt_level != "O2" ]] && [[ $apex_opt_level != "O3" ]]; then + echo "[ERROR] para \"precision_mode\" must be config O1 or O2 or O3" + exit 1 + fi + PREC="--apex --apex-opt-level "$apex_opt_level + +elif [[ $para == --over_dump* ]];then + over_dump=`echo ${para#*=}` + over_dump_path=${cur_path}/output/overflow_dump + mkdir -p ${over_dump_path} + elif [[ $para == --data_dump_flag* ]];then + data_dump_flag=`echo ${para#*=}` + data_dump_path=${cur_path}/output/data_dump + mkdir -p ${data_dump_path} + elif [[ $para == --data_dump_step* ]];then + data_dump_step=`echo ${para#*=}` + elif [[ $para == --profiling* ]];then + profiling=`echo ${para#*=}` + profiling_dump_path=${cur_path}/output/profiling + mkdir -p ${profiling_dump_path} + elif [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + fi +done +#校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be confing" + exit 1 +fi + +#进入训练脚本目录，需要模型审视修改 +cd $cur_path +#创建DeviceID输出目录，不需要修改 +if [ -d ${cur_path}/test/output/${ASCEND_DEVICE_ID} ];then + rm -rf ${cur_path}/test/output/${ASCEND_DEVICE_ID} + mkdir -p ${cur_path}/test/output/$ASCEND_DEVICE_ID/ckpt +else + mkdir -p ${cur_path}/test/output/$ASCEND_DEVICE_ID/ckpt +fi + + +#训练开始时间，不需要修改 +start_time=$(date +%s) +nohup python3 run.py --n-epochs=${train_epochs} --dataset=${data_path} ${PREC} \ + --device_id ${ASCEND_DEVICE_ID} > $cur_path/test/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log 2>&1 & +wait +#训练结束时间，不需要修改 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + + +#结果打印，不需要修改 +echo "------------------ Final result ------------------" +#输出性能FPS，需要模型审视修改 +TrainingTime=`grep "epoch" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |grep -wvE "step:234|step:1|step:2"|awk -F "time: " '{print $2}' |tail -n +3|awk '{sum+=$1} END {print"",sum/NR}'|sed s/[[:space:]]//g` + +FPS=`python3 -c "print(${batch_size}/${TrainingTime})"` +#打印，不需要修改 +echo "Final Performance images/sec : $FPS" + +#打印，不需要修改 +#echo "Final Train Accuracy : ${train_accuracy}" +echo "E2E Training Duration sec : $e2e_time" + +#输出训练精度,需要模型审视修改 +#train_accuracy=`grep "Evluating Epoch" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |awk -F "HR = " '{print $2}'|awk -F ", NDCG" '{print $1}'|awk 'NR==1{max=$1;next}{max=max>$1?max:$1}END{print max}'` + +#性能看护结果汇总 +#训练用例信息，不需要修改 +BatchSize=${batch_size} +DeviceType=`uname -m` +CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'perf' + +##获取性能数据，不需要修改 +#吞吐量 +ActualFPS=${FPS} +#单迭代训练时长 +#TrainingTime=`expr ${batch_size} \* 1000 / ${FPS}` + +#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中，需要根据模型审视 +grep "loss: " $path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log | awk -F "loss: " '{print $2}'|awk -F ", time" '{print $1}' > $path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt + +#最后一个迭代loss值，不需要修改 +ActualLoss=`awk 'END {print}' $path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt` + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${ActualFPS}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log +#echo "TrainAccuracy = ${train_accuracy}" >> $path/output/$ASCEND_DEVICE_ID/${CaseName}.log + diff --git a/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/utils.py b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f7b2d82fc325230c0213386581a6a3292c7a4939 --- /dev/null +++ b/PyTorch/dev/others/RecVAE_ID0347_for_PyTorch/utils.py @@ -0,0 +1,112 @@ +# based on https://github.com/dawenl/vae_cf + +import numpy as np +from scipy import sparse +import pandas as pd +import os +import bottleneck as bn + + + + +def load_train_data(csv_file, n_items, n_users, global_indexing=False): + tp = pd.read_csv(csv_file) + + n_users = n_users if global_indexing else tp['uid'].max() + 1 + + rows, cols = tp['uid'], tp['sid'] + data = sparse.csr_matrix((np.ones_like(rows), + (rows, cols)), dtype='float64', + shape=(n_users, n_items)) + return data + + +def load_tr_te_data(csv_file_tr, csv_file_te, n_items, n_users, global_indexing=False): + tp_tr = pd.read_csv(csv_file_tr) + tp_te = pd.read_csv(csv_file_te) + + if global_indexing: + start_idx = 0 + end_idx = len(unique_uid) - 1 + else: + start_idx = min(tp_tr['uid'].min(), tp_te['uid'].min()) + end_idx = max(tp_tr['uid'].max(), tp_te['uid'].max()) + + rows_tr, cols_tr = tp_tr['uid'] - start_idx, tp_tr['sid'] + rows_te, cols_te = tp_te['uid'] - start_idx, tp_te['sid'] + + data_tr = sparse.csr_matrix((np.ones_like(rows_tr), + (rows_tr, cols_tr)), dtype='float64', shape=(end_idx - start_idx + 1, n_items)) + data_te = sparse.csr_matrix((np.ones_like(rows_te), + (rows_te, cols_te)), dtype='float64', shape=(end_idx - start_idx + 1, n_items)) + return data_tr, data_te + + +def get_data(dataset, global_indexing=False): + unique_sid = list() + with open(os.path.join(dataset, 'unique_sid.txt'), 'r') as f: + for line in f: + unique_sid.append(line.strip()) + + unique_uid = list() + with open(os.path.join(dataset, 'unique_uid.txt'), 'r') as f: + for line in f: + unique_uid.append(line.strip()) + + n_items = len(unique_sid) + n_users = len(unique_uid) + + train_data = load_train_data(os.path.join(dataset, 'train.csv'), n_items, n_users, global_indexing=global_indexing) + + + vad_data_tr, vad_data_te = load_tr_te_data(os.path.join(dataset, 'validation_tr.csv'), + os.path.join(dataset, 'validation_te.csv'), + n_items, n_users, + global_indexing=global_indexing) + + test_data_tr, test_data_te = load_tr_te_data(os.path.join(dataset, 'test_tr.csv'), + os.path.join(dataset, 'test_te.csv'), + n_items, n_users, + global_indexing=global_indexing) + + data = train_data, vad_data_tr, vad_data_te, test_data_tr, test_data_te + data = (x.astype('float32') for x in data) + + return data + + +def ndcg(X_pred, heldout_batch, k=100): + ''' + normalized discounted cumulative gain@k for binary relevance + ASSUMPTIONS: all the 0's in heldout_data indicate 0 relevance + ''' + batch_users = X_pred.shape[0] + idx_topk_part = bn.argpartition(-X_pred, k, axis=1) + topk_part = X_pred[np.arange(batch_users)[:, np.newaxis], + idx_topk_part[:, :k]] + idx_part = np.argsort(-topk_part, axis=1) + # X_pred[np.arange(batch_users)[:, np.newaxis], idx_topk] is the sorted + # topk predicted score + idx_topk = idx_topk_part[np.arange(batch_users)[:, np.newaxis], idx_part] + # build the discount template + tp = 1. / np.log2(np.arange(2, k + 2)) + + DCG = (heldout_batch[np.arange(batch_users)[:, np.newaxis], + idx_topk].toarray() * tp).sum(axis=1) + IDCG = np.array([(tp[:min(n, k)]).sum() + for n in heldout_batch.getnnz(axis=1)]) + return DCG / IDCG + + +def recall(X_pred, heldout_batch, k=100): + batch_users = X_pred.shape[0] + + idx = bn.argpartition(-X_pred, k, axis=1) + X_pred_binary = np.zeros_like(X_pred, dtype=bool) + X_pred_binary[np.arange(batch_users)[:, np.newaxis], idx[:, :k]] = True + + X_true_binary = (heldout_batch > 0).toarray() + tmp = (np.logical_and(X_true_binary, X_pred_binary).sum(axis=1)).astype( + np.float32) + recall = tmp / np.minimum(k, X_true_binary.sum(axis=1)) + return recall \ No newline at end of file