diff --git a/README.md b/README.md index 5db264301d4dc04a901c757cc46b211e941a671e..8afed62d76721ffc1eadeff849248d35f5aa12e7 100644 --- a/README.md +++ b/README.md @@ -663,6 +663,104 @@ ModelsIN用于存放训练获得的分类器 测试主文件为test_SAFC.py +### 3.3 BBDM：基于双向分布匹配的标签移位学习 + +该算法主要包括标签移位数据集构造，BBDM算法实现，性能测试三部分，相关代码参见目录： + +- StreamLearn/Algorithm/Algorithm_BBDM/cifar10_for_labelshift.py +- StreamLearn/Algorithm/Algorithm_BBDM/BBDM.py +- StreamLearn/Algorithm/Algorithm_BBDM/Class_CIFAR10_BBDM.py + + +首先，根据代码配置文件StreamLearn/Algorithm/Algorithm_BBDM/request_import.py安装库完成环境配置 + + +然后，获取算法所需数据集： + +地址： +通过百度网盘分享的文件：cifar-10-python.tar.gz +链接：https://pan.baidu.com/s/1C9BH5QWzwzFTwwxaV-CkXw +提取码：nudt + +说明： +本实验需要用到的数据集是CIFAR10，下载Python格式的数据并存储在StreamLearn//Dataset//data//cifar10/cifar-10-batches-py文件夹中 + +class BBDM_achieve()定义了数据读取、训练、测试的过程，其结构大致如下： +```python +class BBDM_achieve(): + + #初始化 + def __init__(self,arg_address): + self.parser = argparse.ArgumentParser(description='BBDM') + + #数据读取 + def steam_dataread(self): + if (self.args.shift_type == 3) or (self.args.shift_type == 4): + self.alpha = np.ones(10) * self.args.shift_para + self.prob = np.random.dirichlet(self.alpha) + self.shift_para = self.prob + self.shift_para_aux = self.args.shift_para_aux + + else: + self.shift_para = self.args.shift_para + self.shift_para_aux = self.args.shift_para_aux + + + if self.args.data_name == 'cifar10': + self.raw_data = CIFAR10_SHIFT(self.datapath, self.args.training_size, self.args.testing_size, self.args.shift_type, self.shift_para, parameter_aux=self.shift_para_aux, target_label=2, + transform=transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + ]), download=True) + self.D_in = 3072 + if self.args.model == 'MLP': + self.base_model = Net(D_in, 512, 10) + self.train_model = base_model + else: + print('Using Resnet model for predictive tasks') + self.base_model = ResNet18() + self.train_model = self.base_model + self.init_state = self.train_model.state_dict() + else: + raise RuntimeError("Unsupported dataset") + + self.m_train = self.raw_data.get_trainsize() + self.m_valid = self.raw_data.get_validsize() + self.m_test = self.raw_data.get_testsize() + + #训练 + def stream_fit(self): + self.device = torch.device("cpu") + self.kwargs = {'num_workers': 1, 'pin_memory': True} if self.use_cuda else {} + + #测试 + def stream_evaluate(self): + self.test_data = data.Subset(self.raw_data, range(self.m_train, self.m_train+self.m_test)) + self.test_labels = self.raw_data.get_test_label() + self.test_loader = data.DataLoader(self.test_data,batch_size=self.args.batch_size, shuffle=False, **self.kwargs) + self.pre_test, self.acc_test, _, _, self.test_out = test(self.args,self.base_model, self.device, self.test_loader) + self.test_tensor = torch.Tensor(self.test_out) + self.test_label = torch.Tensor(self.test_labels) + self.classes_test = count_classes(self.test_label, self.num_classes) + self.preds_test = torch.softmax(self.test_tensor/self.T + self.b, dim=1) +``` + + +调用函数，读取测试数据并完成测试与性能评估： +```python +def train_and_evaluate_stream_BBDM(args_address,runT): + alg_BBDM=BBDM_achieve(args_address) + for i in range(runT): + alg_BBDM.steam_dataread() + alg_BBDM.stream_fit() + alg_BBDM.stream_evaluate() + return +``` + + +测试主文件为test_BBDM.py ## 课题四 diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/BBDM.py b/StreamLearn/Algorithm/Algorithm_BBDM/BBDM.py new file mode 100644 index 0000000000000000000000000000000000000000..c81e4def9abe18a1b0abd23ab4151e83f3245dd0 --- /dev/null +++ b/StreamLearn/Algorithm/Algorithm_BBDM/BBDM.py @@ -0,0 +1,403 @@ +from __future__ import print_function +import argparse +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import torch.utils.data as data +import torchvision +import torchvision.transforms as transforms +from torchvision import datasets +import numpy as np +import cvxpy as cp +import sklearn.metrics +from sklearn.metrics import precision_recall_fscore_support +import os +import copy +import random +import pandas as pd +import matplotlib.pyplot as plt +from IPython.display import display +from sklearn.metrics import confusion_matrix +import time + +from StreamLearn.Algorithm.Algorithm_BBDM.Gendataloader import GetLoader +from StreamLearn.Algorithm.Algorithm_BBDM.cifar10_for_labelshift import CIFAR10_SHIFT +from StreamLearn.Algorithm.Algorithm_BBDM.resnet import * +from StreamLearn.Algorithm.Algorithm_BBDM.algorithms import * +from StreamLearn.Algorithm.Algorithm_BBDM.KMM import KMM + +class Net(nn.Module): + def __init__(self, D_in, H, D_out): + super(Net, self).__init__() + self.D_in = D_in + self.H = H + self.D_out = D_out + self.model = torch.nn.Sequential( + torch.nn.Linear(self.D_in, self.H), + torch.nn.ReLU(), + torch.nn.Linear(self.H, self.D_out), + ) + + def forward(self, x): + x = x.view(-1, self.D_in) + x = self.model(x) + return x + +class ConvNet(nn.Module): + def __init__(self): + super(ConvNet, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = x.view(-1, 16 * 5 * 5) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + +def train(args, model, device, train_loader, optimizer, epoch, weight=None): + model.train() + #outputsval = np.empty([0,10]) + for batch_idx, (data, target) in enumerate(train_loader): + target = target.type(torch.LongTensor) + target = target.to(device) + data = data.to(device) + optimizer.zero_grad() + output = model(data) + if weight is None: + criterion = nn.CrossEntropyLoss() + else: + criterion = nn.CrossEntropyLoss(weight) + loss = criterion(output, target) + loss.backward() + optimizer.step() + return output + +def test(args, model, device, test_loader, weight=None): + model.eval() + test_loss = 0 + correct = 0 + outs = np.empty([0,1]) + outputsval = np.empty([0,10]) + prediction = np.empty([0,1]) + with torch.no_grad(): + for data, target in test_loader: + target = target.type(torch.LongTensor) + target = target.to(device) + data = data.to(device) + output = model(data) + if weight is None: + criterion = nn.CrossEntropyLoss(reduction='sum') + else: + criterion = nn.CrossEntropyLoss(weight, reduction='sum') + #target = target.type(torch.LongTensor) + loss = criterion(output, target) + outputs = F.softmax(output,dim=1) + test_loss += loss.item()# sum up batch loss + out = outputs.max(1, keepdim=True)[0] + pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability + out = out.cpu().numpy() + correct += pred.eq(target.view_as(pred)).sum().item() + pred = pred.cpu().numpy() + prediction = np.concatenate((prediction, pred)) + outs = np.concatenate((outs, out)) + outputs = outputs.cpu().numpy() + outputsval = np.concatenate((outputsval, output)) + test_loss /= len(test_loader.dataset) + return prediction, 100. * correct / len(test_loader.dataset), test_loss, outs, outputsval + +def count_classes(targets, num_classes): + """ Count number of samples per class in labeled dataset. + + Args: + targets: torch.Tensor (num_data,) with ground truth labels in the dataset + num_classes: int representing number of classes in the dataset + Returns: + counts: torch.Tensor (num_classes, ) with number of samples per class + """ + counts = torch.zeros(num_classes) + for i in range(num_classes): + counts[i] = (targets == i).sum().float() + return counts + +def learn_calibration2(model_outputs, targets, lr, iters, weights): + ''' Implements Bias-Corrected Temperature Scaling (BCTS) from https://arxiv.org/pdf/1901.06852.pdf. + + Code modified from: + https://github.com/gpleiss/temperature_scaling/blob/master/temperature_scaling.py + Args: + model_outputs: torch.Tensor (num_data, num_classes) with outputs of the model before softmax (logits) + targets: torch.Tensor (num_data,) with ground truth labels coresponding to the predictions + lr: float representing learning rate + iters: int specifying number of iterartions + Returns: + T: float with learned temperarture + b: torch.Tensor (num_classes,) with learned biases + ''' + T = torch.tensor([1.], requires_grad=True) + b = torch.ones(model_outputs.shape[1], requires_grad=True) + + nll_criterion = nn.CrossEntropyLoss(weight=weights) + + before_temperature_nll = nll_criterion(model_outputs, targets).item() + + print('Before calibration - NLL: %.3f ' % (before_temperature_nll)) + + optimizer = optim.LBFGS([T, b], lr=lr, max_iter=iters) + def eval(): + loss = nll_criterion(model_outputs/T + b, targets) + loss.backward() + return loss + optimizer.step(eval) + + # Calculate NLL and ECE after temperature scaling + after_temperature_nll = nll_criterion(model_outputs/T + b, targets).item() + print('After calibration - NLL: %.3f ' % (after_temperature_nll)) + return T.item(), b.detach() + + +def compute_true_w(train_labels, test_labels, n_class, m_train, m_test): + # compute the true w + mu_y_train = np.zeros(n_class) + for i in range(n_class): + mu_y_train[i] = float(len(np.where(train_labels == i)[0]))/m_train + mu_y_test = np.zeros(n_class) + for i in range(n_class): + mu_y_test[i] = float(len(np.where(test_labels == i)[0]))/m_test + true_w = mu_y_test/mu_y_train + #print('True w is', true_w) + return true_w + + + +def adjust_predictions(predictions, trainset_priors, test_set_distribution=None): + """ Adjust classifier's predictions to prior shift, + knowing the training set distribution and a different test set distribution. + I.e. predictions are multiplied by the ratio of class priors. + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + trainset_priors: torch.Tensor (num_classes,) + test_set_distribution: torch.Tensor (num_classes,); if None - use uniform distribution + Returns: + adjust_predictions: torch.Tensor (num_data, num_classes) with adjusted predictions + """ + if test_set_distribution is None: + test_set_distribution = torch.ones(trainset_priors.shape) + adjusted_predictions = predictions * test_set_distribution / trainset_priors + adjusted_predictions = adjusted_predictions / torch.sum(adjusted_predictions, dim=1).unsqueeze(1) # normalize to sum to 1 + return adjusted_predictions + +def BBDM_gradient1(predictions, trainset_priors, test_set_distribution=None): + """ Adjust classifier's predictions to prior shift, + knowing the training set distribution and a different test set distribution. + I.e. predictions are multiplied by the ratio of class priors. + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + trainset_priors: torch.Tensor (num_classes,) + test_set_distribution: torch.Tensor (num_classes,); if None - use uniform distribution + Returns: + adjust_predictions: torch.Tensor (num_data, num_classes) with adjusted predictions + """ + if test_set_distribution is None: + test_set_distribution = torch.ones(trainset_priors.shape) + adjusted_prediction = predictions/ trainset_priors + adjusted_predictions = predictions * test_set_distribution / trainset_priors + #adjusted_predictions = adjusted_prediction / torch.sum(adjusted_predictions, dim=1).unsqueeze(1) + adjusted_predictions = adjusted_prediction / torch.sum(adjusted_predictions, dim=1).unsqueeze(1)# normalize to sum to 1 + gra1 = torch.mean(adjusted_predictions, dim=0) # normalize to sum to 1 + return gra1 + +def estimate_priors_from_predictions(predictions): + """ Estimate class priors from predictions. + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + Returns: + priors: torch.Tensor (num_classes) with estimated class priors + """ + + priors = torch.mean(predictions, dim=0) + return priors + + +def next_step_projectedGA_with_prior(x, a, learning_rate, alpha, prior_relative_weight = 1.0): + """ + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + """ + g = compute_gradient(x,a) + g_prior = log_dirichlet_gradient(x, alpha) + nx = x + learning_rate * (g + prior_relative_weight * g_prior) + nx = simplex_projection(nx) + nx = nx / nx.sum() + return nx + +def compute_gradient(x,a): + """ + Compute gradient from Eq. 12 from: + http://openaccess.thecvf.com/content_ICCVW_2019/papers/TASK-CV/Sulc_Improving_CNN_Classifiers_by_Estimating_Test-Time_Priors_ICCVW_2019_paper.pdf + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + """ + d = torch.sum(a*x, dim=1) + g = torch.sum(a*(1/d.unsqueeze(1)), dim=0) + return g + +def log_dirichlet_gradient(x, alpha, numerical_min_prior=1e-8): + """ + Compute gradient from Eq. 15 from: + http://openaccess.thecvf.com/content_ICCVW_2019/papers/TASK-CV/Sulc_Improving_CNN_Classifiers_by_Estimating_Test-Time_Priors_ICCVW_2019_paper.pdf + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + """ + g = (alpha - 1) / torch.max(input=x, other=torch.Tensor([numerical_min_prior])) + return g + +def simplex_projection(y): + """ + Projection onto the probability simplex, based on https://eng.ucmerced.edu/people/wwang5/papers/SimplexProj.pdf + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + """ + u = -np.sort(-y.numpy()) # sort y in descending order + j = np.arange(1, len(y)+1) + phi_obj = u + 1/j * (1-np.cumsum(u)) + positive = np.argwhere(phi_obj > 0) + if positive.size == 0: raise ValueError("Numerical issues - extremely large values after update.. DECREASE LEARNING RATE") + phi = positive.max() + 1 + lam = 1/phi * (1-np.sum(u[:phi])) + x = np.maximum(y+lam,0) + + return torch.Tensor(x) + +def BBDM_estimation(predictions, trainset_priors,K1,K2,K3, termination_difference=0.0001, lam=0.9 ,max_iter=3000, verbose=False): + """ + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + trainset_priors: torch.Tensor (num_classes,) with the train set distribution + test_init_distribution: torch.Tensor (num_classes,) to initialize test set distribution. + If None, use trainset_priors. + termination_error: float defining the distance of posterior predictions for termination. + Returns: + new_predictions: torch.Tensor (num_data, num_classes) with adjusted predictions + new_testset_priors: torch.Tensor (num_classes,) with the estimated test set distribution + """ + E=torch.ones(len(trainset_priors)) + test_init_distribution = trainset_priors.detach().clone() + testset_priors = test_init_distribution / torch.sum(test_init_distribution) + theta = testset_priors + obj0 = (1-lam)*(np.transpose(theta.numpy())@K1 @ theta.numpy() - 2*(K2@ theta.numpy())+K3)-lam*( torch.mean(torch.log(torch.sum(predictions * theta / trainset_priors, dim=1))).numpy()) + + lr = 0.1 + for i in range(int(max_iter)): + theta_int =theta + gra1 = BBDM_gradient1(predictions, trainset_priors, theta) + gra2 = torch.tensor(2*K1 @ theta.numpy() - 2*K2) + gra = (1-lam)*gra2 + lam*(E-gra1) + theta_mid = theta - lr*gra + theta1 = simplex_projection(theta_mid.float()) + obj = (1-lam)*(np.transpose(theta1.numpy())@K1 @ theta1.numpy() - 2*(K2@ theta1.numpy())+K3)-lam*( torch.mean(torch.log(torch.sum(predictions * theta1 / trainset_priors, dim=1))).numpy()) + while obj - obj0 >0 and lr>1e-8: + lr =lr*0.8 + else: + theta = theta1 + obj0 =obj + difference = torch.sum((theta - theta_int)**2) + theta = theta1 + if difference < termination_difference*termination_difference: + #print("Finished. Difference", difference, "< termination value", termination_difference) + break + new_predictions = adjust_predictions(predictions, trainset_priors, theta) + + return new_predictions, theta + + +def label2matrix(label): + label = np.array(label) + uq_la = np.unique(label) + c = uq_la.shape[0] + n = label.shape[0] + label_mat = np.zeros((n,c)) + for i in range(c): + index = (label == i) + label_mat[index,i]=1 + return label_mat + + +def train_validate_test(algpath,args, device, w, train_model, train_loader, test_loader, validate_loader, test_labels, n_class): + w = torch.tensor(w) + #train_model.load_state_dict(init_state) + w = w.float() + + best_loss = 100000 + # model = train_model.to(device)#ConvNet().to(device) + optimizer = optim.SGD(train_model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=5e-4) + for epoch in range(1, args.epochs_training + 1): + train(args, train_model, device, train_loader, optimizer, epoch, weight=w) + # save checkpoint + if epoch > args.epochs_validation: + # validation + _, _, loss,_,_ = test(args, train_model, device, validate_loader, weight=w) + if loss < best_loss: + state = { + 'model': train_model.state_dict(), + } + torch.save(state, algpath+'/checkpoint/ckpt1.pt') + best_loss = loss + checkpoint = torch.load(algpath+'/checkpoint/ckpt1.pt') + train_model.load_state_dict(checkpoint['model']) + predictions, _, _,_,_ = test(args, train_model, device, test_loader) + precision, recall, f1, _ = precision_recall_fscore_support(test_labels, predictions, average='macro') + return precision,recall,f1 + +def train_validate_test2(algpath,args, device, w, train_model, train_loader, test_loader, test_labels, n_class): + w = torch.tensor(w) + #train_model.load_state_dict(init_state) + w = w.float() + + best_loss = 100000 + # model = train_model.to(device)#ConvNet().to(device) + optimizer = optim.SGD(train_model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=5e-4) + for epoch in range(1, args.epochs_estimation + 2): + train(args, train_model, device, train_loader, optimizer, epoch, weight=w) + # save checkpoint + if epoch > args.epochs_validation: + # validation + _, _, loss,_,_ = test(args, train_model, device, test_loader, weight=w) + if loss < best_loss: + state = { + 'model': train_model.state_dict(), + } + torch.save(state, algpath+'/checkpoint/ckpt1.pt') + best_loss = loss + checkpoint = torch.load(algpath+'/checkpoint/ckpt1.pt') + train_model.load_state_dict(checkpoint['model']) + predictions, _, _,_,_ = test(args, train_model, device, test_loader) + precision, recall, f1, _ = precision_recall_fscore_support(test_labels, predictions, average='macro') + return precision,recall,f1 + + + + + + + \ No newline at end of file diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/Class_CIFAR10_BBDM.py b/StreamLearn/Algorithm/Algorithm_BBDM/Class_CIFAR10_BBDM.py new file mode 100644 index 0000000000000000000000000000000000000000..d9dfd54973dd38e45acd9c23d72b25d04dcca711 --- /dev/null +++ b/StreamLearn/Algorithm/Algorithm_BBDM/Class_CIFAR10_BBDM.py @@ -0,0 +1,266 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Dec 23 19:08:42 2024 + +@author: zhangxinyue +""" + +from __future__ import print_function +import argparse +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import torch.utils.data as data +import torchvision +import torchvision.transforms as transforms +from torchvision import datasets +import numpy as np +import cvxpy as cp +import sklearn.metrics +from sklearn.metrics import precision_recall_fscore_support +import os +import copy +import random +import pandas as pd +import matplotlib.pyplot as plt +from IPython.display import display +from sklearn.metrics import confusion_matrix +import time + +from PIL import Image +import os.path +import sys +if sys.version_info[0] == 2: + import cPickle as pickle +else: + import pickle + +from cvxopt import matrix, solvers + +#Ours +from StreamLearn.Algorithm.Algorithm_BBDM.Gendataloader import GetLoader +from StreamLearn.Algorithm.Algorithm_BBDM.cifar10_for_labelshift import CIFAR10_SHIFT +from StreamLearn.Algorithm.Algorithm_BBDM.resnet import * +from StreamLearn.Algorithm.Algorithm_BBDM.algorithms import * +from StreamLearn.Algorithm.Algorithm_BBDM.KMM import KMM +from StreamLearn.Algorithm.Algorithm_BBDM.BBDM import * + +class BBDM_achieve(): + #初始化 + def __init__(self,arg_address): + self.parser = argparse.ArgumentParser(description='BBDM') + self.parser.add_argument('--data-name', type=str, default='cifar10', metavar='N', + help='dataset name, mnist or cifar10 (default: mnist)') + self.parser.add_argument('--training-size', type=int, default=3000, metavar='N', + help='sample size for both training (default: 30000)') + self.parser.add_argument('--testing-size', type=int, default=10000, metavar='N', + help='sample size for testing (default: 30000)') + self.parser.add_argument('--batch-size', type=int, default=64, metavar='N', + help='input batch size for training (default: 64)') + self.parser.add_argument('--test-batch-size', type=int, default=64, metavar='N', + help='input batch size for testing (default: 1000)') + self.parser.add_argument('--shift-type', type = int, default = 3, metavar = 'N', + help = 'Label shift type (default: 2)') + self.parser.add_argument('--shift-para', type = float, default = 0.5, metavar = 'M', + help = 'Label shift paramters (default: 0.2)') + self.parser.add_argument('--shift-para-aux', type = float, default = None, metavar = 'N', + help = 'Label shift paramters (default: 0.2)') + self.parser.add_argument('--model', type = str, default='Resnet', metavar='N', + help = 'model type to use (default MLP)') + self.parser.add_argument('--epochs-estimation', type=int, default=20, metavar='N', + help='number of epochs in weight estimation (default: 40)') + self.parser.add_argument('--epochs-training', type=int, default=25, metavar='N', + help='number of epochs in training (default: 40)') + self.parser.add_argument('--epochs-validation', type=int, default=20, metavar='N', + help='number of epochs before run validation set, smaller than epochs training (default: 10)') + self.parser.add_argument('--lr', type=float, default=0.01, metavar='LR', + help='learning rate (default: 0.01)') + self.parser.add_argument('--momentum', type=float, default=0.5, metavar='M', + help='SGD momentum (default: 0.5)') + self.parser.add_argument('--no-cuda', action='store_true', default=False, + help='disables CUDA training') + self.parser.add_argument('--log-interval', type=int, default=50, metavar='N', + help='how many batches to wait before logging training status') + self.args = self.parser.parse_args() + self.use_cuda = not self.args.no_cuda and torch.cuda.is_available() + self.datapath=arg_address.datapath + self.algpath=arg_address.algpath + #数据读取 + def steam_dataread(self): + if (self.args.shift_type == 3) or (self.args.shift_type == 4): + self.alpha = np.ones(10) * self.args.shift_para + self.prob = np.random.dirichlet(self.alpha) + self.shift_para = self.prob + self.shift_para_aux = self.args.shift_para_aux + + else: + self.shift_para = self.args.shift_para + self.shift_para_aux = self.args.shift_para_aux + + + if self.args.data_name == 'cifar10': + self.raw_data = CIFAR10_SHIFT(self.datapath, self.args.training_size, self.args.testing_size, self.args.shift_type, self.shift_para, parameter_aux=self.shift_para_aux, target_label=2, + transform=transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + ]), download=True) + self.D_in = 3072 + if self.args.model == 'MLP': + self.base_model = Net(D_in, 512, 10) + self.train_model = base_model + else: + print('Using Resnet model for predictive tasks') + self.base_model = ResNet18() + self.train_model = self.base_model + self.init_state = self.train_model.state_dict() + else: + raise RuntimeError("Unsupported dataset") + + self.m_train = self.raw_data.get_trainsize() + self.m_valid = self.raw_data.get_validsize() + self.m_test = self.raw_data.get_testsize() + #训练 + def stream_fit(self): + self.device = torch.device("cpu") + self.kwargs = {'num_workers': 1, 'pin_memory': True} if self.use_cuda else {} + + self.train_data = data.Subset(self.raw_data, range(self.m_train)) + self.valid_data = data.Subset(self.raw_data, range(self.m_train+self.m_test,self.m_train+self.m_test+self.m_valid)) + self.train_labels = self.raw_data.get_train_label() + self.valid_labels = self.raw_data.get_valid_label() + + self.train_loader = data.DataLoader(self.train_data, + batch_size=self.args.batch_size, shuffle=True, **self.kwargs) + self.train2_loader = data.DataLoader(self.train_data, + batch_size=self.args.batch_size, shuffle=False, **self.kwargs) + self.valid_loader = data.DataLoader(self.valid_data, + batch_size=self.args.batch_size, shuffle=False, **self.kwargs) + + self.base_model = self.base_model.to(self.device) + + self.optimizer = optim.SGD(self.base_model.parameters(), lr=self.args.lr, momentum=self.args.momentum, weight_decay=5e-4) + self.best_loss = 100000 + for epoch in range(1, self.args.epochs_training + 1): + self.train_out = train(self.args, self.base_model, self.device, self.train_loader, self.optimizer, epoch) + # save checkpoint + if epoch > self.args.epochs_validation: + # validation + _, _, self.loss,_,_ = test(self.args, self.base_model, self.device, self.valid_loader) + if self.loss < self.best_loss: + print('saving model') + self.state = { + 'model': self.base_model.state_dict(), + } + # if not os.path.isdir('checkpoint'): + # os.mkdir('checkpoint') + torch.save(self.state, self.algpath+'/checkpoint/ckpt.pt') + self.best_loss = self.loss + + self.checkpoint = torch.load(self.algpath+'/checkpoint/ckpt.pt') + self.base_model.load_state_dict(self.checkpoint['model']) + self.pre_train, self.acc_train, _, _, self.train_out = test(self.args, self.base_model, self.device, self.train2_loader) + self.train_tensor = torch.Tensor(self.train_out) + self.pre_valid, self.acc_valid, _, _, self.valid_out = test(self.args, self.base_model, self.device, self.valid_loader) + self.valid_tensor = torch.Tensor(self.valid_out) + + self.train_label = torch.Tensor(self.train_labels) + self.valid_label = torch.Tensor(self.valid_labels) + + self.num_classes = self.train_tensor.shape[1] + self.truep = np.zeros(self.num_classes) + + for i in range(self.num_classes): + self.truep[i] = float(len(np.where(self.train_labels == i)[0]))/self.m_train + + # count samples per class + self.classes_train = count_classes(self.train_label, self.num_classes) + self.classes_val = count_classes(self.valid_label, self.num_classes) + + # Classifier calibration + self.classes_train = self.classes_train / self.classes_train.sum() + self.classes_val = self.classes_val / self.classes_val.sum() + self.weights = self.classes_train / self.classes_val + self.weights = self.weights / self.weights.sum() # class weights to compensate the diffirence in train and val. distributions + + # apply calibration + self.T, self.b = learn_calibration2(self.valid_tensor.float(), self.valid_label.long(), lr=0.0001, iters=1000, weights=self.weights) + self.preds_train = torch.softmax(self.train_tensor/self.T + self.b, dim=1) + self.preds_val = torch.softmax(self.valid_tensor/self.T + self.b, dim=1) + + self.train_soft = torch.softmax(self.train_tensor, dim=1).numpy + self.classifier_priors = torch.mean(self.preds_train, dim=0) + + self.KMM_train = self.raw_data.get_traindata() + self.KMM_train = self.KMM_train.reshape(-1,self.D_in).astype(np.float64) + self.KMM_labels = self.raw_data.get_train_label() + self.KMM_test = self.raw_data.get_testdata() + self.KMM_test = self.KMM_test.reshape(-1,self.D_in).astype(np.float64) + self.kmm = KMM(kernel_type='rbf', gamma=None) + self.D = label2matrix(self.KMM_labels) + self.D = self.D/self.truep + self.K1,self.K2,self.K3 = self.kmm.fit(self.KMM_train, self.KMM_test,self.D,0.001) + #测试 + def stream_evaluate(self): + self.test_data = data.Subset(self.raw_data, range(self.m_train, self.m_train+self.m_test)) + self.test_labels = self.raw_data.get_test_label() + self.test_loader = data.DataLoader(self.test_data,batch_size=self.args.batch_size, shuffle=False, **self.kwargs) + self.pre_test, self.acc_test, _, _, self.test_out = test(self.args,self.base_model, self.device, self.test_loader) + self.test_tensor = torch.Tensor(self.test_out) + self.test_label = torch.Tensor(self.test_labels) + self.classes_test = count_classes(self.test_label, self.num_classes) + self.preds_test = torch.softmax(self.test_tensor/self.T + self.b, dim=1) + + # Prepare structure for results + self.alg_list = ['NA', 'BMDM', 'Oracle'] + self.MSE = dict() + self.Weight =dict() + self.Acc = dict() + self.Fscore =dict() + self.Recall =dict() + self.AAFscore =dict() + self.AARecall =dict() + self.AAWeight =dict() + self.AAMSE = dict() + self.AACC = dict() + #compute weight + self.Weight['Oracle'] = compute_true_w(self.train_labels, self.test_labels, self.num_classes, self.m_train, self.m_test) + + self.lam1 = [0, 0.2, 0.4, 0.6, 0.8, 1] + self.Acc_BBDM =np.zeros(6) + self.AABBDM_min=0 + for i in range(len(self.lam1)):#torch.tensor(truep) + self.pred1_BBDM, self.est_BBDM = BBDM_estimation(self.preds_test, self.classifier_priors,self.K1,self.K2,self.K3, termination_difference=0.0001,lam = self.lam1[i],max_iter=3000) + self.Weight_BMDM = self.est_BBDM.numpy()/self.classifier_priors.numpy() + self.Acc_BBDM[i],_,_ = train_validate_test2(self.algpath,self.args, self.device, self.Weight_BMDM , self.train_model, self.train_loader, self.valid_loader, self.valid_labels, self.num_classes) + if self.Acc_BBDM[i] > self.AABBDM_min: + self.AABBDM_min = self.Acc_BBDM[i] + self.lam_opt = self.lam1[i] + + self.checkpoint = torch.load(self.algpath+'/checkpoint/ckpt.pt') + self.train_model.load_state_dict(self.checkpoint['model']) + self.Acc['NA'],self.Recall['NA'],self.Fscore['NA'] = train_validate_test(self.algpath,self.args, self.device, np.ones(self.num_classes), self.train_model, self.train_loader, self.test_loader, self.valid_loader, self.test_labels, self.num_classes) + + self.pred1_BBDM, self.est_BBDM = BBDM_estimation(self.preds_test, self.classifier_priors,self.K1,self.K2,self.K3, termination_difference=0.0001,lam = self.lam_opt,max_iter=3000) + self.Weight['BMDM'] = self.est_BBDM.numpy()/self.classifier_priors.numpy() + self.checkpoint = torch.load(self.algpath+'/checkpoint/ckpt.pt') + self.train_model.load_state_dict(self.checkpoint['model']) + self.Acc['BMDM'],self.Recall['BMDM'],self.Fscore['BMDM'] = train_validate_test(self.algpath,self.args, self.device, self.Weight['BMDM'], self.train_model, self.train_loader, self.test_loader, self.valid_loader, self.test_labels, self.num_classes) + + self.checkpoint = torch.load(self.algpath+'/checkpoint/ckpt.pt') + self.train_model.load_state_dict(self.checkpoint['model']) + self.Acc['Oracle'],self.Recall['Oracle'],self.Fscore['Oracle'] = train_validate_test(self.algpath,self.args, self.device, self.Weight['Oracle'], self.train_model, self.train_loader, self.test_loader, self.valid_loader, self.test_labels, self.num_classes) + + self.MSE['BMDM'] = np.sum(np.square(self.Weight['Oracle'] - self.Weight['BMDM']))/self.num_classes + + + self.AAMSE =self.MSE + self.AAFscore =self.Fscore + self.AARecall =self.Recall + self.AAWeight =self.Weight + self.AACC = self.Acc + + + \ No newline at end of file diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/Gendataloader.py b/StreamLearn/Algorithm/Algorithm_BBDM/Gendataloader.py new file mode 100644 index 0000000000000000000000000000000000000000..696df822409bb8b14697ff0de48418e5cc99c709 --- /dev/null +++ b/StreamLearn/Algorithm/Algorithm_BBDM/Gendataloader.py @@ -0,0 +1,54 @@ +import torch +import numpy as np +from PIL import Image + + +# 定义GetLoader类，继承Dataset方法，并重写__getitem__()和__len__()方法 +class GetLoader(torch.utils.data.Dataset): + # 初始化函数，得到数据 + def __init__(self, data_root, data_target, data_beta, transform): + self.data = data_root + self.target = torch.LongTensor(data_target) + self.transform = transform + self.beta = data_beta + # index是根据batchsize划分数据后得到的索引，最后将data和对应的labels进行一起返回 + def __getitem__(self, index): + data = self.data[index] + target = self.target[index] + beta = self.beta[index] + + data = Image.fromarray(data.numpy(), mode='L') + + if self.transform is not None: + data = self.transform(data) + + return data, target, beta + # 该函数返回数据大小长度，目的是DataLoader方便划分，如果不知道大小，DataLoader会一脸懵逼 + def __len__(self): + return len(self.data) + + +class GetLoader2(torch.utils.data.Dataset): + # 初始化函数，得到数据 + def __init__(self, data_root, data_target, transform): + self.data = data_root + self.target = torch.LongTensor(data_target) + self.transform = transform + # index是根据batchsize划分数据后得到的索引，最后将data和对应的labels进行一起返回 + def __getitem__(self, index): + data = self.data[index] + target = self.target[index] + + data = Image.fromarray(data.numpy(), mode='L') + #target = torch.LongTensor(target) + if self.transform is not None: + data = self.transform(data) + + return data, target + # 该函数返回数据大小长度，目的是DataLoader方便划分，如果不知道大小，DataLoader会一脸懵逼 + def __len__(self): + return len(self.data) + + + + diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/KMM.py b/StreamLearn/Algorithm/Algorithm_BBDM/KMM.py new file mode 100644 index 0000000000000000000000000000000000000000..27cc6ad4c3dbbf2eec8b11711256764bd3f2ad5a --- /dev/null +++ b/StreamLearn/Algorithm/Algorithm_BBDM/KMM.py @@ -0,0 +1,79 @@ +""" +Kernel Mean Matching +# 1. Gretton, Arthur, et al. "Covariate shift by kernel mean matching." Dataset shift in machine learning 3.4 (2009): 5. +# 2. Huang, Jiayuan, et al. "Correcting sample selection bias by unlabeled data." Advances in neural information processing systems. 2006. +""" + +import numpy as np +import sklearn.metrics +from cvxopt import matrix, solvers +import cvxpy as cp +import os +#rom sklearn.neighbors import KNeighborsClassifier +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('--norm', action='store_true') +args = parser.parse_args() + +def kernel(ker, X1, X2, gamma): + K = None + if ker == 'linear': + if X2 is not None: + K = sklearn.metrics.pairwise.linear_kernel(np.asarray(X1), np.asarray(X2)) + else: + K = sklearn.metrics.pairwise.linear_kernel(np.asarray(X1)) + elif ker == 'rbf': + if X2 is not None: + K = sklearn.metrics.pairwise.rbf_kernel(np.asarray(X1), np.asarray(X2), gamma) + else: + K = sklearn.metrics.pairwise.rbf_kernel(np.asarray(X1), None, gamma) + return K + +class KMM: + def __init__(self, kernel_type='linear', gamma=1): + ''' + Initialization function + :param kernel_type: 'linear' | 'rbf' + :param gamma: kernel bandwidth for rbf kernel + :param B: bound for beta + :param eps: bound for sigma_beta + ''' + self.kernel_type = kernel_type + self.gamma = gamma + #self.alpha = alpha + + def fit(self, Xt, Xs, D, alpha): + ''' + Fit source and target using KMM (compute the coefficients) + :param Xs: ns * dim + :param Xt: nt * dim + :return: Coefficients (Pt / Ps) value vector (Beta in the paper) + ''' + ns = Xs.shape[0] + nt = Xt.shape[0] + Xs = Xs.astype(np.float32) / 255 + Xt = Xt.astype(np.float32) / 255 + Is = np.ones((1,ns)) + Ktt =np.transpose(D)@kernel(self.kernel_type, Xt, None, None)@D/(nt*nt) + Kst = np.sum(kernel(self.kernel_type, Xs, Xt, None)@D/(ns*nt), axis=0) + Kss = Is@kernel(self.kernel_type, Xs, None, None)@np.transpose(Is)/(ns*ns) + #Kst = Kst.T + #Ktt = matrix(Ktt.astype(np.double)) + #Kst = matrix(Kst.astype(np.double)) + #G = matrix(np.r_[np.eye(nt), -np.eye(nt)]) + #h = matrix(np.r_[nt * np.ones((nt,)), np.zeros((nt,))]) + #A = np.ones((1,nt)) + #A = matrix(A) + #b = matrix(np.ones((1,))*nt) + #solvers.options['show_progress'] = False + #sol = solvers.qp(Ktt, -Kst, G, h, A, b, kktsolver='ldl', options={'kktreg':1e-9}) + #theta = np.array(sol['x']) + return Ktt,Kst,Kss + + + # theta = cp.Variable(nt) + # objective = cp.Minimize(theta.T@Ktt@theta/(nt*nt)-2*Is@Kst@theta/(ns*nt) + alpha* cp.sum_squares(theta)) + # constraints = [cp.sum(theta) == nt, theta >=0] + # prob = cp.Problem(objective, constraints) + # result = prob.solve() \ No newline at end of file diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/BBDM.cpython-310.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/BBDM.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..720387dd66e598d98bfb814f0d8abfc6ec77d3d9 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/BBDM.cpython-310.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/Class_CIFAR10_BBDM.cpython-310.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/Class_CIFAR10_BBDM.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..972eccdd2182ccaac05b4c092076289ac2efe5ed Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/Class_CIFAR10_BBDM.cpython-310.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/Gendataloader.cpython-310.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/Gendataloader.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a8969b71e69b000ffd81e36822fcd23bb907174 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/Gendataloader.cpython-310.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/Gendataloader.cpython-37.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/Gendataloader.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7070c5c5c6dfb851c24f37f364d23af093a6b5a3 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/Gendataloader.cpython-37.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/KMM.cpython-310.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/KMM.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..702f67d90c71d93bc63cd51370550e67c02db1d6 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/KMM.cpython-310.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/KMM.cpython-37.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/KMM.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d328a10426c6097311ebc4673c7d1e5560aee85 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/KMM.cpython-37.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/algorithms.cpython-310.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/algorithms.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c81547d587190ffc8765d50226cd747814db7185 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/algorithms.cpython-310.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/algorithms.cpython-37.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/algorithms.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..749b40ab1233e6c4bd2b1106bac18f352e436a97 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/algorithms.cpython-37.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/cifar10_for_labelshift.cpython-310.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/cifar10_for_labelshift.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c113d086c41658d55c4b5f8f4ab3b938e697447b Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/cifar10_for_labelshift.cpython-310.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/mnist_for_labelshift.cpython-310.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/mnist_for_labelshift.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9e22f37b5379015dd0fa761a831615497a181f66 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/mnist_for_labelshift.cpython-310.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/mnist_for_labelshift.cpython-37.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/mnist_for_labelshift.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2661f792ab07a803c873b7aaa881ca944bfdb2a2 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/mnist_for_labelshift.cpython-37.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/resnet.cpython-310.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/resnet.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..94bb43750ccdf52fabc5bcfda87989bb076768c1 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/resnet.cpython-310.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/resnet.cpython-37.pyc b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/resnet.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb79f02a6193fccc0c6a90c286f8b18ed6f4613b Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/__pycache__/resnet.cpython-37.pyc differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/algorithms.py b/StreamLearn/Algorithm/Algorithm_BBDM/algorithms.py new file mode 100644 index 0000000000000000000000000000000000000000..ccdd4a7ea0e55762ce44d3290467142f2823fd78 --- /dev/null +++ b/StreamLearn/Algorithm/Algorithm_BBDM/algorithms.py @@ -0,0 +1,543 @@ +import numpy as np + +import torch +import torch.nn as nn +import torch.optim as optim + +from sklearn.metrics import confusion_matrix + +def hard_confusion_matrix(predictions, targets): + ''' Compute conditional confusion matrix from classifier's predictions + and ground truth labels. + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + targets: torch.Tensor (num_data,) with ground truth labels coresponding to the predictions + Returns: + mat: torch.Tensor (num_classes, num_classes) with hard conditional confusion matrix + ''' + num_classes = predictions.shape[1] + + y = torch.argmax(predictions, dim=1) + + mat = confusion_matrix(targets.numpy(), y.numpy(), normalize='true', labels=np.arange(num_classes)).T + mat = torch.from_numpy(mat) + + return mat.float() + +def soft_confusion_matrix(predictions, targets): + ''' Compute soft confusion matrix from classifier's predictions + and ground truth labels. + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + targets: torch.Tensor (num_data,) with ground truth labels coresponding to the predictions + Returns: + mat: torch.Tensor (num_classes, num_classes) with soft confusion matrix + ''' + num_classes = predictions.shape[1] + + mat = torch.zeros(num_classes, num_classes, dtype=torch.float32) + for i in range(num_classes): + mask = targets == i + m = torch.mean(predictions[mask,:], dim=0) + mat[:,i] = m + + return mat + +def joint_confusion_matrix(predictions, targets, weights): + ''' Compute joint confusion matrix (for BBSE) from classifier's predictions + and ground truth labels. + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + targets: torch.Tensor (num_data,) with ground truth labels coresponding to the predictions + Returns: + mat: torch.Tensor (num_classes, num_classes) with hard conditional confusion matrix + ''' + num_classes = predictions.shape[1] + + y = torch.argmax(predictions, dim=1) + + mat = confusion_matrix(targets.numpy(), y.numpy(), normalize='all', labels=np.arange(num_classes)).T + mat = torch.from_numpy(mat) + + mat = mat*weights + mat = mat / torch.sum(mat) + + return mat.float() + +def compute_joint_soft_confusion_matrix(predictions, targets, weights): + num_classes = predictions.shape[1] + + mat = torch.zeros(num_classes, num_classes, dtype=torch.float32) + for i in range(num_classes): + mask = targets == i + m = torch.mean(predictions[mask,:], dim=0) + mat[:,i] = m + + mat = mat*weights + + return mat / torch.sum(mat) + +def count_classes(targets, num_classes): + """ Count number of samples per class in labeled dataset. + + Args: + targets: torch.Tensor (num_data,) with ground truth labels in the dataset + num_classes: int representing number of classes in the dataset + Returns: + counts: torch.Tensor (num_classes, ) with number of samples per class + """ + counts = torch.zeros(num_classes) + for i in range(num_classes): + counts[i] = (targets == i).sum().float() + return counts + +def accuracy(predictions, gt): + """ Compute accuracy given predictions and ground truth labels. + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + Args: + predictions: torch.Tensor (num_data, num_classes) with output predictions + gt: torch.Tensor (num_data) with ground truth labels. + Returns: + accuracy: float with classifier accuracy + """ + size = gt.shape[0] + predictions = torch.argmax(predictions, dim=1) + acc = torch.sum(predictions == gt).double()/size + return (acc*100).item() + +def adjust_predictions(predictions, trainset_priors, test_set_distribution=None): + """ Adjust classifier's predictions to prior shift, + knowing the training set distribution and a different test set distribution. + I.e. predictions are multiplied by the ratio of class priors. + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + trainset_priors: torch.Tensor (num_classes,) + test_set_distribution: torch.Tensor (num_classes,); if None - use uniform distribution + Returns: + adjust_predictions: torch.Tensor (num_data, num_classes) with adjusted predictions + """ + if test_set_distribution is None: + test_set_distribution = torch.ones(trainset_priors.shape) + adjusted_predictions = predictions * test_set_distribution / trainset_priors + adjusted_predictions = adjusted_predictions / torch.sum(adjusted_predictions, dim=1).unsqueeze(1) # normalize to sum to 1 + return adjusted_predictions + +def simplex_projection(y): + """ + Projection onto the probability simplex, based on https://eng.ucmerced.edu/people/wwang5/papers/SimplexProj.pdf + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + """ + u = -np.sort(-y.numpy()) # sort y in descending order + j = np.arange(1, len(y)+1) + phi_obj = u + 1/j * (1-np.cumsum(u)) + positive = np.argwhere(phi_obj > 0) + if positive.size == 0: raise ValueError("Numerical issues - extremely large values after update.. DECREASE LEARNING RATE") + phi = positive.max() + 1 + lam = 1/phi * (1-np.sum(u[:phi])) + x = np.maximum(y+lam,0) + + return torch.Tensor(x) + +############### +# Calibration # +############### + +def learn_calibration(model_outputs, targets, lr, iters, weights): + ''' Implements Bias-Corrected Temperature Scaling (BCTS) from https://arxiv.org/pdf/1901.06852.pdf. + + Code modified from: + https://github.com/gpleiss/temperature_scaling/blob/master/temperature_scaling.py + Args: + model_outputs: torch.Tensor (num_data, num_classes) with outputs of the model before softmax (logits) + targets: torch.Tensor (num_data,) with ground truth labels coresponding to the predictions + lr: float representing learning rate + iters: int specifying number of iterartions + Returns: + T: float with learned temperarture + b: torch.Tensor (num_classes,) with learned biases + ''' + T = torch.tensor([1.], requires_grad=True) + b = torch.ones(model_outputs.shape[1], requires_grad=True) + + nll_criterion = nn.CrossEntropyLoss(weight=weights) + + before_temperature_nll = nll_criterion(model_outputs, targets).item() + + print('Before calibration - NLL: %.3f ' % (before_temperature_nll)) + + optimizer = optim.LBFGS([T, b], lr=lr, max_iter=iters) + + def eval(): + loss = nll_criterion(model_outputs/T + b, targets) + loss.backward() + return loss + optimizer.step(eval) + + # Calculate NLL and ECE after temperature scaling + after_temperature_nll = nll_criterion(model_outputs/T + b, targets).item() + print('After calibration - NLL: %.3f ' % (after_temperature_nll)) + + return T.item(), b.detach() + +################ +# EM algorithm # +################ + +def estimate_priors_from_predictions(predictions): + """ Estimate class priors from predictions. + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + Returns: + priors: torch.Tensor (num_classes) with estimated class priors + """ + + priors = torch.mean(predictions, dim=0) + return priors + + +def EM_priors_estimation(predictions, trainset_priors, test_init_distribution=None, termination_difference=0.0001, verbose=False): + """ EM algorithm for test set prior estimation and adjust classifier's predictions + to prior shift. + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + trainset_priors: torch.Tensor (num_classes,) with the train set distribution + test_init_distribution: torch.Tensor (num_classes,) to initialize test set distribution. + If None, use trainset_priors. + termination_error: float defining the distance of posterior predictions for termination. + Returns: + new_predictions: torch.Tensor (num_data, num_classes) with adjusted predictions + new_testset_priors: torch.Tensor (num_classes,) with the estimated test set distribution + """ + if test_init_distribution is None: + test_init_distribution = trainset_priors.detach().clone() + + testset_priors = test_init_distribution / torch.sum(test_init_distribution) + step = 0 + + while True: + step += 1 + new_predictions = adjust_predictions(predictions, trainset_priors, testset_priors) + new_testset_priors = estimate_priors_from_predictions(new_predictions) + + difference = torch.sum((new_testset_priors - testset_priors)**2) + if verbose: print("EM step ", step, "; diff. %.8f" % (difference)) + if difference < termination_difference*termination_difference: + if verbose: print("Finished. Difference", difference, "< termination value", termination_difference) + break + testset_priors = new_testset_priors + + return new_predictions, new_testset_priors + +######################### +# MLE and MAP Estimate # +######################### + +# Projected Gradient Ascent for ML estimate is applied by iteratively running next_step_projectedGA() +# Projected Gradient Ascent for MAP estimate is applied by iteratively running next_step_projectedGA_with_prior() + +def compute_gradient(x,a): + """ + Compute gradient from Eq. 12 from: + http://openaccess.thecvf.com/content_ICCVW_2019/papers/TASK-CV/Sulc_Improving_CNN_Classifiers_by_Estimating_Test-Time_Priors_ICCVW_2019_paper.pdf + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + """ + d = torch.sum(a*x, dim=1) + g = torch.sum(a*(1/d.unsqueeze(1)), dim=0) + return g + +def log_dirichlet_gradient(x, alpha, numerical_min_prior=1e-8): + """ + Compute gradient from Eq. 15 from: + http://openaccess.thecvf.com/content_ICCVW_2019/papers/TASK-CV/Sulc_Improving_CNN_Classifiers_by_Estimating_Test-Time_Priors_ICCVW_2019_paper.pdf + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + """ + g = (alpha - 1) / torch.max(input=x, other=torch.Tensor([numerical_min_prior])) + return g + +def next_step_projectedGA(x, a, learning_rate): + """ + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + """ + g = compute_gradient(x,a) + nx = x + learning_rate * g + nx = simplex_projection(nx) + nx = nx / nx.sum() + return nx + +def next_step_projectedGA_with_prior(x, a, learning_rate, alpha, prior_relative_weight = 1.0): + """ + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + """ + g = compute_gradient(x,a) + g_prior = log_dirichlet_gradient(x, alpha) + nx = x + learning_rate * (g + prior_relative_weight * g_prior) + nx = simplex_projection(nx) + nx = nx / nx.sum() + return nx + +def MLE_estimate(predictions, trainset_priors, num_iter, test_init_distribution=None, lr=1e-7, termination_difference=0.0001): + ''' + Maximum likelihood estimate according to + http://openaccess.thecvf.com/content_ICCVW_2019/papers/TASK-CV/Sulc_Improving_CNN_Classifiers_by_Estimating_Test-Time_Priors_ICCVW_2019_paper.pdf + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + trainset_priors: torch.Tensor (num_classes,) with the train set distribution + num_iter: int max. number of iterations + test_init_distribution: torch.Tensor (num_classes,) to initialize test set distribution. + If None, use trainset_priors. + termination_difference: float defining the distance of posterior predictions for termination. + Returns: + new_predictions: torch.Tensor (num_data, num_classes) with adjusted predictions + new_testset_priors: torch.Tensor (num_classes,) with the estimated test set distribution + ''' + mask = (trainset_priors == 0) + a = predictions/torch.where(mask, torch.ones_like(trainset_priors), trainset_priors) + a[:,mask] = 0 + + if test_init_distribution is None: + testset_priors = trainset_priors.detach().clone() + else: + testset_priors = test_init_distribution + testset_priors = testset_priors / torch.sum(testset_priors) + + for iteration in range(int(num_iter)): + new_testset_priors = next_step_projectedGA(testset_priors, a, learning_rate=lr) + + difference = torch.sum((new_testset_priors - testset_priors)**2) + if difference < termination_difference*termination_difference: + break + testset_priors = new_testset_priors + + new_predictions = adjust_predictions(predictions, trainset_priors, testset_priors) + return new_predictions, new_testset_priors + +def MAP_estimate(predictions, trainset_priors, num_iter, test_init_distribution=None, lr=1e-8, termination_difference=0.0001, alpha=3): + ''' + Maximum aposteriori estimate according to + http://openaccess.thecvf.com/content_ICCVW_2019/papers/TASK-CV/Sulc_Improving_CNN_Classifiers_by_Estimating_Test-Time_Priors_ICCVW_2019_paper.pdf + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + trainset_priors: torch.Tensor (num_classes,) with the train set distribution + test_init_distribution: torch.Tensor (num_classes,) to initialize test set distribution. + If None, use trainset_priors. + termination_difference: float defining the distance of posterior predictions for termination. + Returns: + new_predictions: torch.Tensor (num_data, num_classes) with adjusted predictions + new_testset_priors: torch.Tensor (num_classes,) with the estimated test set distribution + ''' + mask = (trainset_priors == 0) + a = predictions/torch.where(mask, torch.ones_like(trainset_priors), trainset_priors) + a[:,mask] = 0 + + if test_init_distribution is None: + map_priors = trainset_priors.detach().clone() + else: + map_priors = test_init_distribution + map_priors = map_priors / torch.sum(map_priors) + + for iteration in range(int(num_iter)): + testset_priors = next_step_projectedGA_with_prior(map_priors, a, alpha=alpha, learning_rate=lr) + + difference = torch.sum((testset_priors - map_priors)**2) + if difference < termination_difference*termination_difference: + break + map_priors = testset_priors + + new_predictions = adjust_predictions(predictions, trainset_priors, map_priors) + return new_predictions, testset_priors + + +########################### +# CONFUSION MATRIX BASED # +########################### + +def CM_estimate(predictions, confusion_matrix, soft=False): + ''' + Test set prior estimation using confusion matrix inversion (Equation 4 in the paper). + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + confusion_matrix: torch.Tensor (num_classes, num_classes) + soft: bool indicator for soft confusion matrix + Returns: + new_testset_priors: torch.Tensor (num_classes,) with the estimated test set distribution + ''' + num_classes = predictions.shape[1] + + if not soft: + decision_distribution = torch.zeros(num_classes) + class_pred = torch.argmax(predictions, dim=1) + for i in range(num_classes): + decision_distribution[i] = torch.sum(class_pred == i) + else: + decision_distribution = torch.sum(predictions, dim=0) + + decision_distribution = decision_distribution / torch.sum(decision_distribution) + + if torch.matrix_rank(confusion_matrix) == num_classes: + conf_inv = torch.inverse(confusion_matrix) + else: + conf_inv = torch.pinverse(confusion_matrix) + + priors_estimate = (conf_inv @ decision_distribution.unsqueeze(1)).squeeze() + priors_estimate[priors_estimate < 0] = 0 + priors_estimate = priors_estimate / torch.sum(priors_estimate) + + return priors_estimate + +def BBSE_estimate(predictions, confusion_matrix, soft=False): + ''' + Test set prior estimation using confusion matrix inversion (Equation 4 in the paper). + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + confusion_matrix: torch.Tensor (num_classes, num_classes) + soft: bool indicator for soft confusion matrix + Returns: + w: torch.Tensor (num_classes,) estimated priors ratio + ''' + num_classes = predictions.shape[1] + + if not soft: + decision_distribution = torch.zeros(num_classes) + class_pred = torch.argmax(predictions, dim=1) + for i in range(num_classes): + decision_distribution[i] = torch.sum(class_pred == i) + else: + decision_distribution = torch.sum(predictions, dim=0) + + decision_distribution = decision_distribution / torch.sum(decision_distribution) + + if torch.matrix_rank(confusion_matrix) == num_classes: + conf_inv = torch.inverse(confusion_matrix) + else: + conf_inv = torch.pinverse(confusion_matrix) + + w = (conf_inv @ decision_distribution.unsqueeze(1)).squeeze() + + return w + + +def matrix_correction_MLE(predictions, trainset_priors, confusion_matrix, soft=False, max_iter=1000, lr=1e-3): + ''' + Maximum likelihood estimation of test set priors using confusion matrix, proposed in Section 3.1. + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + trainset_priors: torch.Tensor (num_classes,) with the train set distribution + confusion_matrix: torch.Tensor (num_classes, num_classes) + soft: bool indicator for soft condusion matrix + max_iter: int max. number of iterations + lr: float learning rate + Returns: + new_predictions: torch.Tensor (num_data, num_classes) with adjusted predictions + new_testset_priors: torch.Tensor (num_classes,) with the estimated test set distribution + ''' + d = 2 # number to divide lr + num = 50 # lr will be devided each num iterations by d + num_classes = predictions.shape[1] + + if not soft: + decision_distribution = torch.zeros(num_classes) + class_pred = torch.argmax(predictions, dim=1) + for i in range(num_classes): + decision_distribution[i] = torch.sum(class_pred == i) + else: + decision_distribution = torch.sum(predictions, dim=0) + + decision_distribution = decision_distribution / torch.sum(decision_distribution) + + new_testset_priors = trainset_priors.detach().clone() + new_testset_priors = new_testset_priors/torch.sum(new_testset_priors) + for i in range(int(max_iter)): + o = confusion_matrix @ new_testset_priors + mask = (o == 0) + grad = decision_distribution / torch.where(mask, torch.ones_like(o), o) + grad[mask] = 0 + + grad = confusion_matrix*grad.unsqueeze(1) + grad = grad.sum(0) + + lr_cur = lr/d**(i//num) + p_updated = new_testset_priors + lr_cur*grad + new_testset_priors = simplex_projection(p_updated) + + + new_predictions = adjust_predictions(predictions, trainset_priors, new_testset_priors) + + return new_predictions, new_testset_priors + +def matrix_correction_MAP(predictions, trainset_priors, confusion_matrix, alpha=3, soft=False, max_iter=1000, lr=0.01): + ''' + Maximum a-posteriori estimation of test set priors with a Dirichlet hyperprior, using confusion matrix, + proposed in Section 3.2. + Args: + predictions: torch.Tensor (num_data, num_classes) with predictions + trainset_priors: torch.Tensor (num_classes,) with the train set distribution + confusion_matrix: torch.Tensor (num_classes, num_classes) to initialize test set distribution. + If None, use trainset_priors. + alpha: float hyperprior of dirichlet distribution + soft: bool indicator for soft condusion matrix + max_iter: int max. number of iterations + lr: float learning rate + Returns: + new_predictions: torch.Tensor (num_data, num_classes) with adjusted predictions + new_testset_priors: torch.Tensor (num_classes,) with the estimated test set distribution + ''' + num_classes = predictions.shape[1] + + if not soft: + decision_distribution = torch.zeros(num_classes) + class_pred = torch.argmax(predictions, dim=1) + for i in range(num_classes): + decision_distribution[i] = torch.sum(class_pred == i) + else: + decision_distribution = torch.sum(predictions, dim=0) + + N = torch.sum(decision_distribution) + decision_distribution = decision_distribution / N + + new_testset_priors = trainset_priors.detach().clone() + + new_testset_priors = new_testset_priors/torch.sum(new_testset_priors) + for i in range(int(max_iter)): + o = confusion_matrix @ new_testset_priors + mask = (o == 0) + grad_l = decision_distribution/torch.where(mask, torch.ones_like(o), o) + grad_l[mask] = 0 + + grad_l = confusion_matrix*grad_l.unsqueeze(1) + grad_l = grad_l.sum(0) + + grad_a = ((alpha-1)/new_testset_priors) + + grad = grad_a/N + grad_l # divide N because decision_distribution are normalized + + p_updated = new_testset_priors + lr*grad + new_testset_priors = simplex_projection(p_updated) + + new_predictions = adjust_predictions(predictions, trainset_priors, new_testset_priors) + + return new_predictions, new_testset_priors diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/checkpoint/ckpt.pt b/StreamLearn/Algorithm/Algorithm_BBDM/checkpoint/ckpt.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e927fc821a2be59876b697171ad7d9da236fcb7 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/checkpoint/ckpt.pt differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/checkpoint/ckpt1.pt b/StreamLearn/Algorithm/Algorithm_BBDM/checkpoint/ckpt1.pt new file mode 100644 index 0000000000000000000000000000000000000000..3efbddb16b0510d292a4154ceb6ceb842ca06546 Binary files /dev/null and b/StreamLearn/Algorithm/Algorithm_BBDM/checkpoint/ckpt1.pt differ diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/cifar10_for_labelshift.py b/StreamLearn/Algorithm/Algorithm_BBDM/cifar10_for_labelshift.py new file mode 100644 index 0000000000000000000000000000000000000000..57083b648c3334c198a86b10130d5e95f2aad58b --- /dev/null +++ b/StreamLearn/Algorithm/Algorithm_BBDM/cifar10_for_labelshift.py @@ -0,0 +1,362 @@ +from __future__ import print_function +from PIL import Image +import os +import os.path +import numpy as np +import sys +if sys.version_info[0] == 2: + import cPickle as pickle +else: + import pickle +import torch +import torch.utils.data as data +from torchvision import datasets + + +class CIFAR10_SHIFT(data.Dataset): + """`CIFAR10 `_ Dataset. + Args: + root (string): Root directory of dataset where directory + ``cifar-10-batches-py`` exists or will be saved to if download is set to True. + train (bool, optional): If True, creates dataset from training set, otherwise + creates from test set. + transform (callable, optional): A function/transform that takes in an PIL image + and returns a transformed version. E.g, ``transforms.RandomCrop`` + target_transform (callable, optional): A function/transform that takes in the + target and transforms it. + download (bool, optional): If true, downloads the dataset from the internet and + puts it in root directory. If dataset is already downloaded, it is not + downloaded again. + """ + base_folder = 'cifar-10-batches-py' + url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" + filename = "cifar-10-python.tar.gz" + tgz_md5 = 'c58f30108f718f92721af3b95e74349a' + train_list = [ + ['data_batch_1', 'c99cafc152244af753f735de768cd75f'], + ['data_batch_2', 'd4bba439e000b95fd0a9bffe97cbabec'], + ['data_batch_3', '54ebc095f3ab1f0389bbae665268c751'], + ['data_batch_4', '634d18415352ddfa80567beed471001a'], + ['data_batch_5', '482c414d41f54cd18b22e5b47cb7c3cb'], + ] + + test_list = [ + ['test_batch', '40351d587109b95175f43aff81a1287e'], + ] + meta = { + 'filename': 'batches.meta', + 'key': 'label_names', + 'md5': '5ff9c542aee3614f3951f8cda6e48888', + } + + @property + def targets(self): + return self.labels + + + def __init__(self, root, training_size, testing_size, shift_type, parameter, parameter_aux=None, target_label=None, + transform=None, target_transform=None, + download=False): + """ + Converted function from original torch cifar10 class : + new parameters: + sample_size: int, sample size of both source and target set + shift_type: int, 1 for knock one shift on source, target is uniform + 2 for tweak one shift on source, target is uniform + 3 for dirichlet shift on source, target is uniform + 4 for dirichlet shift on the target set, source is uniform + 5 for tweak one shift on the target set, source is uniform + 6 for Minority class shift on source, target is uniform + 7 for tweak on shift on both source and target + parameter: float in [0, 1], delta for knock one shift, delete target_label by delta + or, rho for tweak one shift, set target_label probability as rho, others even + or, alpha for dirichlet shift + or, # of minority class for Minority class shift + or, rho for tweak one shift on source for 7 + parameter_aux: float in [0, 1], required for 6 and 7 + 6, the proportion of data for minority class + 7, rho for tweak one shift on target for 7 + target_label: int, target label for knock one and tweak one shift (1,2,5) + """ + self.root = os.path.expanduser(root) + self.transform = transform + self.target_transform = target_transform + + if download: + self.download() + + if not self._check_integrity(): + raise RuntimeError('Dataset not found or corrupted.' + + ' You can use download=True to download it') + + # now load the picked numpy arrays + # merge the training and testing together + + raw_data = [] + raw_labels = [] + for fentry in self.train_list: + f = fentry[0] + file = os.path.join(self.root, self.base_folder, f) + fo = open(file, 'rb') + if sys.version_info[0] == 2: + entry = pickle.load(fo) + else: + entry = pickle.load(fo, encoding='latin1') + raw_data.append(entry['data']) + if 'labels' in entry: + raw_labels += entry['labels'] + else: + raw_labels += entry['fine_labels'] + fo.close() + + + f = self.test_list[0][0] + file = os.path.join(self.root, self.base_folder, f) + fo = open(file, 'rb') + if sys.version_info[0] == 2: + entry = pickle.load(fo) + else: + entry = pickle.load(fo, encoding='latin1') + raw_data.append(entry['data']) + if 'labels' in entry: + raw_labels += entry['labels'] + else: + raw_labels += entry['fine_labels'] + fo.close() + + self._load_meta() + # merge training and testing + raw_data = np.concatenate(raw_data) + raw_labels = np.asarray(raw_labels) + # # creat label shift + # indices = np.random.permutation(60000) + features = raw_data + labels = raw_labels + + indices = np.random.permutation(60000) + ec_train = int((50000-testing_size)/10) + ec_test = int(testing_size/10) + ec_valid = 100 + + + + indices_trains = np.empty((0,1), dtype = int) + indices_tests = np.empty((0,1), dtype = int) + indices_valids = np.empty((0,1), dtype = int) + + for j in range(10): + indices_al = np.where(labels == j)[0] + shuffle = np.random.permutation(len(indices_al)) + indices_all=indices_al[shuffle] + indices_tr = indices_all[0:ec_train] + indices_trains = np.append(indices_trains, indices_tr) + indices_te = indices_all[ec_train:ec_train+ec_test] + indices_tests = np.append(indices_tests, indices_te) + indices_val = indices_all[ec_train+ec_test:ec_train+ec_test+ec_valid] + indices_valids = np.append(indices_valids, indices_val) + + test_data = features[(indices_tests,)] + test_labels = labels[(indices_tests,)] + train_data = features[(indices_trains,)] + train_labels = labels[(indices_trains,)] + valid_data = features[(indices_valids,)] + valid_labels = labels[(indices_valids,)] + + if shift_type == 2: + if target_label == None: + raise RuntimeError("There should be a target label for the tweak one shift.") + # use the number of target label to decide the total number of the training samples + if parameter < (1.0-parameter)/9 : + target_label = (target_label + 1)%10 + indices_target = np.where(train_labels == target_label)[0] + num_target1 = len(indices_target) + num_train = int(num_target1/parameter) + if num_train > training_size: + num_train = training_size + num_remain = num_train*(1-parameter) + num_target = int(num_train*parameter) + # even on other labels + num_i = int(num_remain/9) + + indices_train = np.empty((0,1), dtype = int) + + for i in range(10): + indices_i = np.where(train_labels == i)[0] + if i != target_label: + indices_i = indices_i[0:num_i] + else: + num_tar = training_size - 9*num_i + indices_i = indices_i[0:num_tar] + indices_train = np.append(indices_train, indices_i) + + shuffle = np.random.permutation(len(indices_train)) + + train_data = train_data[(indices_train[shuffle],)] + train_labels = train_labels[(indices_train[shuffle],)] + + elif shift_type == 3: + # use the maximum prob to decide the total number of training samples + target_label = np.argmax(parameter) + + indices_target = np.where(train_labels == target_label)[0] + num_target = len(indices_target) + + prob_max = np.amax(parameter) + num_train = int(num_target/prob_max) + if num_train > training_size: + num_train = training_size + indices_train = np.empty((0,1), dtype = int) + + for i in range(target_label): + num_i = int(num_train * parameter[i]) + if num_i <40: + num_i=40 + indices_i = np.where(train_labels == i)[0] + indices_i = indices_i[0:num_i] + indices_train = np.append(indices_train, indices_i) + for i in range(target_label+1,10): + num_i = int(num_train * parameter[i]) + if num_i <40: + num_i=40 + indices_i = np.where(train_labels == i)[0] + indices_i = indices_i[0:num_i] + indices_train = np.append(indices_train, indices_i) + num_max = training_size - len(indices_train) + indices_tar = indices_target[0:num_max] + indices_train = np.append(indices_train, indices_tar) + + shuffle = np.random.permutation(len(indices_train)) + train_data = train_data[(indices_train[shuffle],)] + train_labels = train_labels[(indices_train[shuffle],)] + else: + raise RuntimeError("Invalid shift type.") + + indices_train = np.empty((0,1), dtype = int) + indices_valid = np.empty((0,1), dtype = int) + for j in range(10): + indices_al = np.where(train_labels == j)[0] + ec_train = int(len(indices_al)*8/10) + ec_val = int(len(indices_al)*2/10) + shuffle = np.random.permutation(len(indices_al)) + indices_all=indices_al[shuffle] + indices_tr = indices_all[0:ec_train] + indices_train = np.append(indices_train, indices_tr) + indices_va = indices_all[ec_train:len(indices_al)] + indices_valid = np.append(indices_valid, indices_va) + valid_data = train_data[(indices_valid,)] + valid_labels = train_labels[(indices_valid,)] + train_data = train_data[(indices_train,)] + train_labels = train_labels[(indices_train,)] + +#.reshape((-1, 3, 32, 32)) .transpose((0, 2, 3, 1)) + self.data = ((np.concatenate(( train_data,test_data,valid_data))).reshape((-1, 3, 32, 32))).transpose((0, 2, 3, 1)) + self.labels = np.concatenate((train_labels,test_labels,valid_labels)) + self.train_labels = train_labels + self.test_labels = test_labels + self.valid_labels = valid_labels + self.m_train = len(train_labels) + self.m_test = len(test_labels) + self.m_valid = len(valid_labels) + self.train_data = (train_data.reshape((-1, 3, 32, 32))).transpose((0, 2, 3, 1)) + self.test_data = (test_data.reshape((-1, 3, 32, 32))).transpose((0, 2, 3, 1)) + + + + + def _load_meta(self): + path = os.path.join(self.root, self.base_folder, self.meta['filename']) + if not datasets.utils.check_integrity(path, self.meta['md5']): + raise RuntimeError('Dataset metadata file not found or corrupted.' + + ' You can use download=True to download it') + with open(path, 'rb') as infile: + if sys.version_info[0] == 2: + data = pickle.load(infile) + else: + data = pickle.load(infile, encoding='latin1') + self.classes = data[self.meta['key']] + self.class_to_idx = {_class: i for i, _class in enumerate(self.classes)} + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (image, target) where target is index of the target class. + """ + + img, target = self.data[index], self.labels[index] + + + # doing this so that it is consistent with all other datasets + # to return a PIL Image + img = Image.fromarray(img) + + if self.transform is not None: + img = self.transform(img) + + if self.target_transform is not None: + target = self.target_transform(target) + + return img, target + + def __len__(self): + return len(self.data) + + def get_train_label(self): + return self.train_labels + def get_test_label(self): + return self.test_labels + def get_valid_label(self): + return self.valid_labels + def get_data(self): + return self.data + def get_label(self): + return self.labels + def get_trainsize(self): + return self.m_train + def get_testsize(self): + return self.m_test + def get_validsize(self): + return self.m_valid + def get_traindata(self): + return self.train_data + def get_testdata(self): + return self.test_data + + + def _check_integrity(self): + root = self.root + for fentry in (self.train_list + self.test_list): + filename, md5 = fentry[0], fentry[1] + fpath = os.path.join(root, self.base_folder, filename) + if not datasets.utils.check_integrity(fpath, md5): + return False + return True + + def download(self): + import tarfile + + if self._check_integrity(): + print('Files already downloaded and verified') + return + + root = self.root + datasets.utils.download_url(self.url, root, self.filename, self.tgz_md5) + + # extract file + cwd = os.getcwd() + tar = tarfile.open(os.path.join(root, self.filename), "r:gz") + os.chdir(root) + tar.extractall() + tar.close() + os.chdir(cwd) + + def __repr__(self): + fmt_str = 'Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + fmt_str += ' Root Location: {}\n'.format(self.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/request_import.py b/StreamLearn/Algorithm/Algorithm_BBDM/request_import.py new file mode 100644 index 0000000000000000000000000000000000000000..9ab20bcd32304432b1ebae5c5d4dc8ebf0ddb826 --- /dev/null +++ b/StreamLearn/Algorithm/Algorithm_BBDM/request_import.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Dec 23 18:52:11 2024 + +@author: zhangxinyue +""" + +#本实验需要用到的库 +from __future__ import print_function +import argparse +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import torch.utils.data as data +import torchvision +import torchvision.transforms as transforms +from torchvision import datasets +import numpy as np +import cvxpy as cp +import sklearn.metrics +from sklearn.metrics import precision_recall_fscore_support +import os +import copy +import random +import pandas as pd +import matplotlib.pyplot as plt +from IPython.display import display +from sklearn.metrics import confusion_matrix +import time + +from PIL import Image +import os.path +import sys +if sys.version_info[0] == 2: + import cPickle as pickle +else: + import pickle + +from cvxopt import matrix, solvers + +from types import SimpleNamespace + +#Ours +from StreamLearn.Algorithm.Algorithm_BBDM.Gendataloader import GetLoader +from StreamLearn.Algorithm.Algorithm_BBDM.cifar10_for_labelshift import CIFAR10_SHIFT +from StreamLearn.Algorithm.Algorithm_BBDM.resnet import * +from StreamLearn.Algorithm.Algorithm_BBDM.algorithms import * +from StreamLearn.Algorithm.Algorithm_BBDM.KMM import KMM +from StreamLearn.Algorithm.Algorithm_BBDM.BBDM import * +from StreamLearn.Algorithm.Algorithm_BBDM.Class_CIFAR10_BBDM import * diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/resnet.py b/StreamLearn/Algorithm/Algorithm_BBDM/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..07cf44a05a969dc8e15523d3fb50a432ad3c00d7 --- /dev/null +++ b/StreamLearn/Algorithm/Algorithm_BBDM/resnet.py @@ -0,0 +1,152 @@ +'''ResNet in PyTorch. +Reference: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 +''' +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion*planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = F.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(ResNet, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 64, num_blocks[1], stride=2) + # self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2) + # self.layer4 = self._make_layer(block, 64, num_blocks[3], stride=2) + self.linear = nn.Linear(1024*block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1]*(num_blocks-1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + # out = self.layer3(out) + # out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + +class ResNet2(nn.Module): + def __init__(self, block, num_blocks, num_classes=100): + super(ResNet2, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 64, num_blocks[1], stride=2) + # self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2) + # self.layer4 = self._make_layer(block, 64, num_blocks[3], stride=2) + self.linear = nn.Linear(1024*block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1]*(num_blocks-1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + # out = self.layer3(out) + # out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + +def ResNet18(): + return ResNet(BasicBlock, [2,2,2,2]) + +def ResNet182(): + return ResNet2(BasicBlock, [2,2,2,2]) + +def ResNet34(): + return ResNet(BasicBlock, [3,4,6,3]) + +def ResNet50(): + return ResNet(Bottleneck, [3,4,6,3]) + +def ResNet101(): + return ResNet(Bottleneck, [3,4,23,3]) + +def ResNet152(): + return ResNet(Bottleneck, [3,8,36,3]) + + +def test(): + net = ResNet18() + y = net(torch.randn(1,3,32,32)) + print(y.size()) + +# test() \ No newline at end of file diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/sample.py b/StreamLearn/Algorithm/Algorithm_BBDM/sample.py new file mode 100644 index 0000000000000000000000000000000000000000..1e098e3d13501d87a05475c59854787393830ac3 --- /dev/null +++ b/StreamLearn/Algorithm/Algorithm_BBDM/sample.py @@ -0,0 +1,36 @@ +import prior_shift +import torch +import numpy as np + + +def accuracy(predictions, gt): + """ Compute accuracy given predictions and ground truth labels. + Args: + predictions: np.array (num_data, num_classes) with output predictions + gt: np.array (num_data) with ground truth labels. + Returns: + accuracy: float with classifier accuracy + """ + size = gt.shape[0] + predictions = np.argmax(predictions, axis=1) + acc = np.sum(predictions == gt).astype(float)/size + return (acc*100).item() + +predictions_dict = torch.load('CIFAR100_LT_outputs/outputs.pth.tar', map_location=torch.device('cpu')) +outputs_train = predictions_dict['outputs_train'].detach().to('cpu') +targets_train = predictions_dict['targets_train'].detach().to('cpu').numpy() +outputs_val = predictions_dict['outputs_val'].detach().to('cpu') +targets_val = predictions_dict['targets_val'].detach().to('cpu').numpy() +outputs_test = predictions_dict['outputs_test'].detach().to('cpu') +targets_test = predictions_dict['targets_test'].detach().to('cpu').numpy() + +preds_train = torch.softmax(outputs_train, dim=1).numpy() +preds_val = torch.softmax(outputs_val, dim=1).numpy() +preds_test = torch.softmax(outputs_test, dim=1).numpy() + +preds = prior_shift.adapt_to_prior_shift(preds_test, preds_val, targets_val, preds_train, targets_train) +acc_after = accuracy(preds, targets_test) +acc_before = accuracy(preds_test, targets_test) +print('Accuracy before: ' ,acc_before, ' accuracy after: ', acc_after) + + diff --git a/StreamLearn/Algorithm/Algorithm_BBDM/tools.py b/StreamLearn/Algorithm/Algorithm_BBDM/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..1b30fda22838fae5fbb06aa456839a3e8089da07 --- /dev/null +++ b/StreamLearn/Algorithm/Algorithm_BBDM/tools.py @@ -0,0 +1,76 @@ +import numpy as np +from sklearn.metrics import confusion_matrix + +def adjust_predictions(predictions, trainset_priors, test_set_distribution=None): + """ Adjust classifier's predictions to prior shift, + knowing the training set distribution and a different test set distribution. + I.e. predictions are multiplied by the ratio of class priors. + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + Args: + predictions: np.array (num_data, num_classes) with predictions + trainset_priors: np.array (num_classes,) + test_set_distribution: np.array (num_classes,); if None - use uniform distribution + Returns: + adjusted_predictions: np.array (num_data, num_classes) with adjusted predictions + """ + if test_set_distribution is None: + test_set_distribution = np.ones(trainset_priors.shape) + adjusted_predictions = predictions * test_set_distribution / trainset_priors + adjusted_predictions = adjusted_predictions / np.expand_dims(np.sum(adjusted_predictions, axis=1), 1) # normalize to sum to 1 + return adjusted_predictions + +def simplex_projection(y): + """ + Projection onto the probability simplex, based on https://eng.ucmerced.edu/people/wwang5/papers/SimplexProj.pdf + + Code modified from: + https://github.com/sulc/priors-example/blob/master/cifar-priors-example.ipynb + """ + u = -np.sort(-y) # sort y in descending order + j = np.arange(1, len(y)+1) + phi_obj = u + 1/j * (1-np.cumsum(u)) + positive = np.argwhere(phi_obj > 0) + if positive.size == 0: raise ValueError("Numerical issues - extremely large values after update.. DECREASE LEARNING RATE") + phi = positive.max() + 1 + lam = 1/phi * (1-np.sum(u[:phi])) + x = np.maximum(y+lam,0) + + return x + +def hard_confusion_matrix(predictions, targets): + ''' Compute conditional confusion matrix from classifier's predictions + and ground truth labels. + Args: + predictions: np.array (num_data, num_classes) with predictions + targets: np.array (num_data,) with ground truth labels coresponding to the predictions + Returns: + mat: np.array (num_classes, num_classes) with hard conditional confusion matrix + ''' + num_classes = predictions.shape[1] + + y = np.argmax(predictions, axis=1) + + mat = confusion_matrix(targets, y, normalize='true', labels=np.arange(num_classes)).T + + return mat.astype(float) + +def soft_confusion_matrix(predictions, targets): + ''' Compute soft confusion matrix from classifier's predictions + and ground truth labels. + Args: + predictions: np.array (num_data, num_classes) with predictions + targets: np.array (num_data,) with ground truth labels coresponding to the predictions + Returns: + mat: np.array (num_classes, num_classes) with soft confusion matrix + ''' + num_classes = predictions.shape[1] + + mat = np.zeros((num_classes, num_classes), dtype=float) + for i in range(num_classes): + mask = targets == i + m = np.mean(predictions[mask,:], axis=0) + mat[:,i] = m + + return mat \ No newline at end of file diff --git a/StreamLearn/Dataset/data/.keep b/StreamLearn/Dataset/data/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/StreamLearn/Dataset/data/cifar10/.keep b/StreamLearn/Dataset/data/cifar10/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/StreamLearn/Dataset/data/cifar10/cifar-10-batches-py/.keep b/StreamLearn/Dataset/data/cifar10/cifar-10-batches-py/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/StreamLearn/tests/test_BBDM.py b/StreamLearn/tests/test_BBDM.py new file mode 100644 index 0000000000000000000000000000000000000000..299eb5cda667a32b26b918850566c8c1f74e33f1 --- /dev/null +++ b/StreamLearn/tests/test_BBDM.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Dec 23 19:11:30 2024 + +@author: zhangxinyue +""" + +from __future__ import print_function +import argparse +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import torch.utils.data as data +import torchvision +import torchvision.transforms as transforms +from torchvision import datasets +import numpy as np +import cvxpy as cp +import sklearn.metrics +from sklearn.metrics import precision_recall_fscore_support +import os +import copy +import random +import pandas as pd +import matplotlib.pyplot as plt +from IPython.display import display +from sklearn.metrics import confusion_matrix +import time + +from PIL import Image +import os.path +import sys +if sys.version_info[0] == 2: + import cPickle as pickle +else: + import pickle + +from cvxopt import matrix, solvers + +from types import SimpleNamespace + +#Ours +from StreamLearn.Algorithm.Algorithm_BBDM.Gendataloader import GetLoader +from StreamLearn.Algorithm.Algorithm_BBDM.cifar10_for_labelshift import CIFAR10_SHIFT +from StreamLearn.Algorithm.Algorithm_BBDM.resnet import * +from StreamLearn.Algorithm.Algorithm_BBDM.algorithms import * +from StreamLearn.Algorithm.Algorithm_BBDM.KMM import KMM +from StreamLearn.Algorithm.Algorithm_BBDM.BBDM import * +from StreamLearn.Algorithm.Algorithm_BBDM.Class_CIFAR10_BBDM import * + +#%% +def train_and_evaluate_stream_BBDM(args_address,runT): + alg_BBDM=BBDM_achieve(args_address) + for i in range(runT): + alg_BBDM.steam_dataread() + alg_BBDM.stream_fit() + alg_BBDM.stream_evaluate() + return + +def main(): + args_address=SimpleNamespace(algpath="StreamLearn/Algorithm/Algorithm_BBDM",datapath="StreamLearn//Dataset//data//cifar10") + runT=1 + train_and_evaluate_stream_BBDM(args_address,runT) + +if __name__ == "__main__": + main() + + \ No newline at end of file