From 517546ac0315e0f52903c9d5809792761ef12564 Mon Sep 17 00:00:00 2001 From: darrylong Date: Fri, 29 Sep 2023 16:22:18 +0800 Subject: [PATCH 01/23] Generated model base from LightGCN --- cornac/models/__init__.py | 1 + cornac/models/ngcf/__init__.py | 16 ++ cornac/models/ngcf/ngcf.py | 102 ++++++++++ cornac/models/ngcf/recom_ngcf.py | 292 ++++++++++++++++++++++++++++ cornac/models/ngcf/requirements.txt | 2 + examples/ngcf_example.py | 58 ++++++ 6 files changed, 471 insertions(+) create mode 100644 cornac/models/ngcf/__init__.py create mode 100644 cornac/models/ngcf/ngcf.py create mode 100644 cornac/models/ngcf/recom_ngcf.py create mode 100644 cornac/models/ngcf/requirements.txt create mode 100644 examples/ngcf_example.py diff --git a/cornac/models/__init__.py b/cornac/models/__init__.py index 4674a36db..82b2a9e0d 100644 --- a/cornac/models/__init__.py +++ b/cornac/models/__init__.py @@ -51,6 +51,7 @@ from .ncf import GMF from .ncf import MLP from .ncf import NeuMF +from .ngcf import NGCF from .nmf import NMF from .online_ibpr import OnlineIBPR from .pcrl import PCRL diff --git a/cornac/models/ngcf/__init__.py b/cornac/models/ngcf/__init__.py new file mode 100644 index 000000000..ab703d890 --- /dev/null +++ b/cornac/models/ngcf/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2018 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from .recom_ngcf import NGCF diff --git a/cornac/models/ngcf/ngcf.py b/cornac/models/ngcf/ngcf.py new file mode 100644 index 000000000..22983ab3b --- /dev/null +++ b/cornac/models/ngcf/ngcf.py @@ -0,0 +1,102 @@ +import torch +import torch.nn as nn +import dgl +import dgl.function as fn + + +def construct_graph(data_set): + """ + Generates graph given a cornac data set + + Parameters + ---------- + data_set : cornac.data.dataset.Dataset + The data set as provided by cornac + """ + user_indices, item_indices, _ = data_set.uir_tuple + user_nodes, item_nodes = ( + torch.from_numpy(user_indices), + torch.from_numpy( + item_indices + data_set.total_users + ), # increment item node idx by num users + ) + + u = torch.cat([user_nodes, item_nodes], dim=0) + v = torch.cat([item_nodes, user_nodes], dim=0) + + g = dgl.graph((u, v), num_nodes=(data_set.total_users + data_set.total_items)) + return g + + +class GCNLayer(nn.Module): + def __init__(self): + super(GCNLayer, self).__init__() + + def forward(self, graph, src_embedding, dst_embedding): + with graph.local_scope(): + inner_product = torch.cat((src_embedding, dst_embedding), dim=0) + + out_degs = graph.out_degrees().to(src_embedding.device).float().clamp(min=1) + norm_out_degs = torch.pow(out_degs, -0.5).view(-1, 1) # D^-1/2 + + inner_product = inner_product * norm_out_degs + + graph.ndata["h"] = inner_product + graph.update_all( + message_func=fn.copy_u("h", "m"), reduce_func=fn.sum("m", "h") + ) + + res = graph.ndata["h"] + + in_degs = graph.in_degrees().to(src_embedding.device).float().clamp(min=1) + norm_in_degs = torch.pow(in_degs, -0.5).view(-1, 1) # D^-1/2 + + res = res * norm_in_degs + return res + + +class Model(nn.Module): + def __init__(self, user_size, item_size, hidden_size, num_layers=3, device=None): + super(Model, self).__init__() + self.user_size = user_size + self.item_size = item_size + self.hidden_size = hidden_size + self.embedding_weights = self._init_weights() + self.layers = nn.ModuleList([GCNLayer() for _ in range(num_layers)]) + self.device = device + + def forward(self, graph): + user_embedding = self.embedding_weights["user_embedding"] + item_embedding = self.embedding_weights["item_embedding"] + + for i, layer in enumerate(self.layers, start=1): + if i == 1: + embeddings = layer(graph, user_embedding, item_embedding) + else: + embeddings = layer( + graph, embeddings[: self.user_size], embeddings[self.user_size:] + ) + + user_embedding = user_embedding + embeddings[: self.user_size] * ( + 1 / (i + 1) + ) + item_embedding = item_embedding + embeddings[self.user_size:] * ( + 1 / (i + 1) + ) + + return user_embedding, item_embedding + + def _init_weights(self): + initializer = nn.init.xavier_uniform_ + + weights_dict = nn.ParameterDict( + { + "user_embedding": nn.Parameter( + initializer(torch.empty(self.user_size, self.hidden_size)) + ), + "item_embedding": nn.Parameter( + initializer(torch.empty(self.item_size, self.hidden_size)) + ), + } + ) + return weights_dict diff --git a/cornac/models/ngcf/recom_ngcf.py b/cornac/models/ngcf/recom_ngcf.py new file mode 100644 index 000000000..48a0658ad --- /dev/null +++ b/cornac/models/ngcf/recom_ngcf.py @@ -0,0 +1,292 @@ +# Copyright 2018 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from ..recommender import Recommender +from ...exception import ScoreException + +from tqdm.auto import tqdm, trange + + +class NGCF(Recommender): + """ + NGCF + + Parameters + ---------- + name: string, default: 'NGCF' + The name of the recommender model. + + num_epochs: int, default: 1000 + Maximum number of iterations or the number of epochs + + learning_rate: float, default: 0.001 + The learning rate that determines the step size at each iteration + + train_batch_size: int, default: 1024 + Mini-batch size used for train set + + test_batch_size: int, default: 100 + Mini-batch size used for test set + + hidden_dim: int, default: 64 + The embedding size of the model + + num_layers: int, default: 3 + Number of LightGCN Layers + + early_stopping: {min_delta: float, patience: int}, optional, default: None + If `None`, no early stopping. Meaning of the arguments: + + - `min_delta`: the minimum increase in monitored value on validation + set to be considered as improvement, + i.e. an increment of less than min_delta will count as + no improvement. + + - `patience`: number of epochs with no improvement after which + training should be stopped. + + lambda_reg: float, default: 1e-4 + Weight decay for the L2 normalization + + trainable: boolean, optional, default: True + When False, the model is not trained and Cornac assumes that the model + is already pre-trained. + + verbose: boolean, optional, default: False + When True, some running logs are displayed. + + seed: int, optional, default: 2020 + Random seed for parameters initialization. + + References + ---------- + * He, X., Deng, K., Wang, X., Li, Y., Zhang, Y., & Wang, M. (2020). + LightGCN: Simplifying and Powering Graph Convolution Network for + Recommendation. + """ + + def __init__( + self, + name="NGCF", + num_epochs=1000, + learning_rate=0.001, + train_batch_size=1024, + test_batch_size=100, + hidden_dim=64, + num_layers=3, + early_stopping=None, + lambda_reg=1e-4, + trainable=True, + verbose=False, + seed=2020, + ): + super().__init__(name=name, trainable=trainable, verbose=verbose) + + self.num_epochs = num_epochs + self.learning_rate = learning_rate + self.hidden_dim = hidden_dim + self.num_layers = num_layers + self.train_batch_size = train_batch_size + self.test_batch_size = test_batch_size + self.early_stopping = early_stopping + self.lambda_reg = lambda_reg + self.seed = seed + + def fit(self, train_set, val_set=None): + """Fit the model to observations. + + Parameters + ---------- + train_set: :obj:`cornac.data.Dataset`, required + User-Item preference data as well as additional modalities. + + val_set: :obj:`cornac.data.Dataset`, optional, default: None + User-Item preference data for model selection purposes (e.g., early stopping). + + Returns + ------- + self : object + """ + Recommender.fit(self, train_set, val_set) + + if not self.trainable: + return self + + # model setup + import torch + from .lightgcn import Model + from .lightgcn import construct_graph + + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + if self.seed is not None: + torch.manual_seed(self.seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(self.seed) + + model = Model( + train_set.total_users, + train_set.total_items, + self.hidden_dim, + self.num_layers, + ).to(self.device) + + graph = construct_graph(train_set).to(self.device) + + optimizer = torch.optim.Adam( + model.parameters(), lr=self.learning_rate, weight_decay=self.lambda_reg + ) + loss_fn = torch.nn.BCELoss(reduction="sum") + + # model training + pbar = trange( + self.num_epochs, + desc="Training", + unit="iter", + position=0, + leave=False, + disable=not self.verbose, + ) + for _ in pbar: + model.train() + accum_loss = 0.0 + for batch_u, batch_i, batch_j in tqdm( + train_set.uij_iter( + batch_size=self.train_batch_size, + shuffle=True, + ), + desc="Epoch", + total=train_set.num_batches(self.train_batch_size), + leave=False, + position=1, + disable=not self.verbose, + ): + user_embeddings, item_embeddings = model(graph) + + batch_u = torch.from_numpy(batch_u).long().to(self.device) + batch_i = torch.from_numpy(batch_i).long().to(self.device) + batch_j = torch.from_numpy(batch_j).long().to(self.device) + + user_embed = user_embeddings[batch_u] + positive_item_embed = item_embeddings[batch_i] + negative_item_embed = item_embeddings[batch_j] + + ui_scores = (user_embed * positive_item_embed).sum(dim=1) + uj_scores = (user_embed * negative_item_embed).sum(dim=1) + + loss = loss_fn( + torch.sigmoid(ui_scores - uj_scores), torch.ones_like(ui_scores) + ) + accum_loss += loss.cpu().item() + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + accum_loss /= len(train_set.uir_tuple[0]) # normalize over all observations + pbar.set_postfix(loss=accum_loss) + + # store user and item embedding matrices for prediction + model.eval() + self.U, self.V = model(graph) + + if self.early_stopping is not None and self.early_stop( + **self.early_stopping + ): + break + + # we will use numpy for faster prediction in the score function, no need torch + self.U = self.U.cpu().detach().numpy() + self.V = self.V.cpu().detach().numpy() + + def monitor_value(self): + """Calculating monitored value used for early stopping on validation set (`val_set`). + This function will be called by `early_stop()` function. + + Returns + ------- + res : float + Monitored value on validation set. + Return `None` if `val_set` is `None`. + """ + if self.val_set is None: + return None + + import torch + + loss_fn = torch.nn.BCELoss(reduction="sum") + accum_loss = 0.0 + pbar = tqdm( + self.val_set.uij_iter(batch_size=self.test_batch_size), + desc="Validation", + total=self.val_set.num_batches(self.test_batch_size), + leave=False, + position=1, + disable=not self.verbose, + ) + for batch_u, batch_i, batch_j in pbar: + batch_u = torch.from_numpy(batch_u).long().to(self.device) + batch_i = torch.from_numpy(batch_i).long().to(self.device) + batch_j = torch.from_numpy(batch_j).long().to(self.device) + + user_embed = self.U[batch_u] + positive_item_embed = self.V[batch_i] + negative_item_embed = self.V[batch_j] + + ui_scores = (user_embed * positive_item_embed).sum(dim=1) + uj_scores = (user_embed * negative_item_embed).sum(dim=1) + + loss = loss_fn( + torch.sigmoid(ui_scores - uj_scores), torch.ones_like(ui_scores) + ) + accum_loss += loss.cpu().item() + pbar.set_postfix(val_loss=accum_loss) + + accum_loss /= len(self.val_set.uir_tuple[0]) + return -accum_loss # higher is better -> smaller loss is better + + def score(self, user_idx, item_idx=None): + """Predict the scores/ratings of a user for an item. + + Parameters + ---------- + user_idx: int, required + The index of the user for whom to perform score prediction. + + item_idx: int, optional, default: None + The index of the item for which to perform score prediction. + If None, scores for all known items will be returned. + + Returns + ------- + res : A scalar or a Numpy array + Relative scores that the user gives to the item or to all known items + + """ + if item_idx is None: + if self.train_set.is_unk_user(user_idx): + raise ScoreException( + "Can't make score prediction for (user_id=%d)" % user_idx + ) + known_item_scores = self.V.dot(self.U[user_idx, :]) + return known_item_scores + else: + if self.train_set.is_unk_user(user_idx) or self.train_set.is_unk_item( + item_idx + ): + raise ScoreException( + "Can't make score prediction for (user_id=%d, item_id=%d)" + % (user_idx, item_idx) + ) + return self.V[item_idx, :].dot(self.U[user_idx, :]) diff --git a/cornac/models/ngcf/requirements.txt b/cornac/models/ngcf/requirements.txt new file mode 100644 index 000000000..32f294fbc --- /dev/null +++ b/cornac/models/ngcf/requirements.txt @@ -0,0 +1,2 @@ +torch>=2.0.0 +dgl>=1.1.0 \ No newline at end of file diff --git a/examples/ngcf_example.py b/examples/ngcf_example.py new file mode 100644 index 000000000..f4ee49941 --- /dev/null +++ b/examples/ngcf_example.py @@ -0,0 +1,58 @@ +# Copyright 2018 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Example for NGCF, using the CiteULike dataset +""" +import cornac +from cornac.datasets import citeulike +from cornac.eval_methods import RatioSplit + +# Load user-item feedback +data = citeulike.load_feedback() + +# Instantiate an evaluation method to split data into train and test sets. +ratio_split = RatioSplit( + data=data, + val_size=0.1, + test_size=0.1, + exclude_unknowns=True, + verbose=True, + seed=123, + rating_threshold=0.5, +) + +# Instantiate the NGCF model +lightgcn = cornac.models.NGCF( + seed=123, + num_epochs=2000, + num_layers=3, + early_stopping={"min_delta": 1e-4, "patience": 3}, + train_batch_size=256, + learning_rate=0.001, + lambda_reg=1e-4, + verbose=True +) + +# Instantiate evaluation measures +rec_20 = cornac.metrics.Recall(k=20) +ndcg_20 = cornac.metrics.NDCG(k=20) + +# Put everything together into an experiment and run it +cornac.Experiment( + eval_method=ratio_split, + models=[lightgcn], + metrics=[rec_20, ndcg_20], + user_based=True, +).run() From 61ad3c9c5f9a1d2c562b418298ae2d4c1508a875 Mon Sep 17 00:00:00 2001 From: darrylong Date: Mon, 2 Oct 2023 17:51:34 +0800 Subject: [PATCH 02/23] wip --- cornac/models/ngcf/ngcf.py | 79 ++++++++++++++++++++++---------- cornac/models/ngcf/recom_ngcf.py | 8 ++-- 2 files changed, 58 insertions(+), 29 deletions(-) diff --git a/cornac/models/ngcf/ngcf.py b/cornac/models/ngcf/ngcf.py index 22983ab3b..db6ff1bb1 100644 --- a/cornac/models/ngcf/ngcf.py +++ b/cornac/models/ngcf/ngcf.py @@ -29,73 +29,102 @@ def construct_graph(data_set): class GCNLayer(nn.Module): - def __init__(self): + def __init__(self, in_size, out_size, dropout): super(GCNLayer, self).__init__() + self.in_size = in_size + self.out_size = out_size + + self.w1 = nn.Linear(in_size, out_size, bias=True) + self.w2 = nn.Linear(in_size, out_size, bias=True) - def forward(self, graph, src_embedding, dst_embedding): + self.leaky_relu = nn.LeakyReLU(0.2) + + self.dropout = nn.Dropout(dropout) + + torch.nn.init.xavier_uniform_(self.w1.weight) + torch.nn.init.constant_(self.w1.bias, 0) + + torch.nn.init.xavier_uniform_(self.w2.weight) + torch.nn.init.constant_(self.w2.bias, 0) + + def forward(self, graph, norm, src_embedding, dst_embedding): with graph.local_scope(): inner_product = torch.cat((src_embedding, dst_embedding), dim=0) - out_degs = graph.out_degrees().to(src_embedding.device).float().clamp(min=1) - norm_out_degs = torch.pow(out_degs, -0.5).view(-1, 1) # D^-1/2 + in_degs, out_degs = graph.edges() + # out_degs = graph.out_degrees().to(src_embedding.device).float().clamp(min=1) + + msgs = self.w1(inner_product[in_degs]) + self.w2(inner_product[in_degs] * inner_product[out_degs]) + # norm_msgs = torch.pow(msgs, -0.5).view(-1, 1) # D^-1/2 + msgs = norm * msgs - inner_product = inner_product * norm_out_degs + # inner_product = inner_product * norm_msgs - graph.ndata["h"] = inner_product + graph.edata["h"] = msgs + # graph.ndata["h"] = inner_product graph.update_all( - message_func=fn.copy_u("h", "m"), reduce_func=fn.sum("m", "h") + message_func=fn.copy_e("h", "m"), reduce_func=fn.sum("m", "h") ) res = graph.ndata["h"] + res = self.leaky_relu(res) + res = self.dropout(res) - in_degs = graph.in_degrees().to(src_embedding.device).float().clamp(min=1) - norm_in_degs = torch.pow(in_degs, -0.5).view(-1, 1) # D^-1/2 + # in_degs = graph.in_degrees().to(src_embedding.device).float().clamp(min=1) + # norm_in_degs = torch.pow(in_degs, -0.5).view(-1, 1) # D^-1/2 - res = res * norm_in_degs + # res = res * norm_in_degs return res class Model(nn.Module): - def __init__(self, user_size, item_size, hidden_size, num_layers=3, device=None): + def __init__(self, user_size, item_size, embed_size=64, layer_size=[64, 64, 64], dropout=[0.1, 0.1, 0.1], device=None): super(Model, self).__init__() self.user_size = user_size self.item_size = item_size - self.hidden_size = hidden_size - self.embedding_weights = self._init_weights() - self.layers = nn.ModuleList([GCNLayer() for _ in range(num_layers)]) + self.embedding_weights = self._init_weights(embed_size) + self.layers = nn.ModuleList([GCNLayer(embed_size, layer_size[0], dropout[0])]) + self.layers.extend( + [ + GCNLayer( + layer_size[i], layer_size[i + 1], dropout=dropout[i + 1] + ) for i in range(len(layer_size) - 1) + ] + ) self.device = device def forward(self, graph): user_embedding = self.embedding_weights["user_embedding"] item_embedding = self.embedding_weights["item_embedding"] + src, dst = graph.edges() + dst_degree = graph.in_degrees(dst).float() + src_degree = graph.out_degrees(src).float() + norm = torch.pow(src_degree * dst_degree, -0.5).view(-1, 1) # D^-1/2 + for i, layer in enumerate(self.layers, start=1): if i == 1: - embeddings = layer(graph, user_embedding, item_embedding) + embeddings = layer(graph, norm, user_embedding, item_embedding) else: embeddings = layer( - graph, embeddings[: self.user_size], embeddings[self.user_size:] + graph, norm, embeddings[: self.user_size], embeddings[self.user_size:] ) - user_embedding = user_embedding + embeddings[: self.user_size] * ( - 1 / (i + 1) - ) - item_embedding = item_embedding + embeddings[self.user_size:] * ( - 1 / (i + 1) - ) + user_embedding = user_embedding + embeddings[: self.user_size] + item_embedding = item_embedding + embeddings[self.user_size:] return user_embedding, item_embedding - def _init_weights(self): + def _init_weights(self, in_size): initializer = nn.init.xavier_uniform_ weights_dict = nn.ParameterDict( { "user_embedding": nn.Parameter( - initializer(torch.empty(self.user_size, self.hidden_size)) + initializer(torch.empty(self.user_size, in_size)) ), "item_embedding": nn.Parameter( - initializer(torch.empty(self.item_size, self.hidden_size)) + initializer(torch.empty(self.item_size, in_size)) ), } ) diff --git a/cornac/models/ngcf/recom_ngcf.py b/cornac/models/ngcf/recom_ngcf.py index 48a0658ad..a2be47c58 100644 --- a/cornac/models/ngcf/recom_ngcf.py +++ b/cornac/models/ngcf/recom_ngcf.py @@ -126,8 +126,8 @@ def fit(self, train_set, val_set=None): # model setup import torch - from .lightgcn import Model - from .lightgcn import construct_graph + from .ngcf import Model + from .ngcf import construct_graph self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") if self.seed is not None: @@ -138,8 +138,8 @@ def fit(self, train_set, val_set=None): model = Model( train_set.total_users, train_set.total_items, - self.hidden_dim, - self.num_layers, + # self.hidden_dim, + # self.num_layers, ).to(self.device) graph = construct_graph(train_set).to(self.device) From 93d677f2be76438113241e7dbaecf687b8877fc3 Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Tue, 3 Oct 2023 11:51:01 +0800 Subject: [PATCH 03/23] wip example --- examples/ngcf_example.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/examples/ngcf_example.py b/examples/ngcf_example.py index f4ee49941..ebc3cbc81 100644 --- a/examples/ngcf_example.py +++ b/examples/ngcf_example.py @@ -33,8 +33,19 @@ rating_threshold=0.5, ) +lightgcn = cornac.models.LightGCN( + seed=123, + num_epochs=2000, + num_layers=3, + early_stopping={"min_delta": 1e-4, "patience": 3}, + train_batch_size=256, + learning_rate=0.001, + lambda_reg=1e-4, + verbose=True +) + # Instantiate the NGCF model -lightgcn = cornac.models.NGCF( +ngcf = cornac.models.NGCF( seed=123, num_epochs=2000, num_layers=3, @@ -45,6 +56,10 @@ verbose=True ) +gcmc = cornac.models.GCMC( + seed=123, +) + # Instantiate evaluation measures rec_20 = cornac.metrics.Recall(k=20) ndcg_20 = cornac.metrics.NDCG(k=20) @@ -52,7 +67,7 @@ # Put everything together into an experiment and run it cornac.Experiment( eval_method=ratio_split, - models=[lightgcn], + models=[lightgcn, ngcf], metrics=[rec_20, ndcg_20], user_based=True, ).run() From 30fdf12a1949f4eeb90ea985f821297f6a1ba6d8 Mon Sep 17 00:00:00 2001 From: tqtg Date: Sat, 7 Oct 2023 06:20:10 +0000 Subject: [PATCH 04/23] add self-connection --- cornac/models/ngcf/ngcf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cornac/models/ngcf/ngcf.py b/cornac/models/ngcf/ngcf.py index db6ff1bb1..a2f4ac2b0 100644 --- a/cornac/models/ngcf/ngcf.py +++ b/cornac/models/ngcf/ngcf.py @@ -51,10 +51,10 @@ def forward(self, graph, norm, src_embedding, dst_embedding): with graph.local_scope(): inner_product = torch.cat((src_embedding, dst_embedding), dim=0) - in_degs, out_degs = graph.edges() + srt, dst = graph.edges() # out_degs = graph.out_degrees().to(src_embedding.device).float().clamp(min=1) - msgs = self.w1(inner_product[in_degs]) + self.w2(inner_product[in_degs] * inner_product[out_degs]) + msgs = self.w1(inner_product[srt]) + self.w2(inner_product[srt] * inner_product[dst]) # norm_msgs = torch.pow(msgs, -0.5).view(-1, 1) # D^-1/2 msgs = norm * msgs @@ -66,7 +66,7 @@ def forward(self, graph, norm, src_embedding, dst_embedding): message_func=fn.copy_e("h", "m"), reduce_func=fn.sum("m", "h") ) - res = graph.ndata["h"] + res = self.w1(inner_product) + graph.ndata["h"] res = self.leaky_relu(res) res = self.dropout(res) From e3b1650b7bb69acf83058a719fc32713cc46ec09 Mon Sep 17 00:00:00 2001 From: tqtg Date: Sun, 8 Oct 2023 23:21:15 +0000 Subject: [PATCH 05/23] refactor code --- cornac/models/ngcf/ngcf.py | 235 +++++++++++++++++++------------ cornac/models/ngcf/recom_ngcf.py | 122 ++++++---------- examples/ngcf_example.py | 29 ++-- 3 files changed, 195 insertions(+), 191 deletions(-) diff --git a/cornac/models/ngcf/ngcf.py b/cornac/models/ngcf/ngcf.py index a2f4ac2b0..6d25d47eb 100644 --- a/cornac/models/ngcf/ngcf.py +++ b/cornac/models/ngcf/ngcf.py @@ -1,9 +1,16 @@ +# Reference: https://github.com/dmlc/dgl/blob/master/examples/pytorch/NGCF/NGCF/model.py + import torch import torch.nn as nn +import torch.nn.functional as F import dgl import dgl.function as fn +USER_KEY = "user" +ITEM_KEY = "item" + + def construct_graph(data_set): """ Generates graph given a cornac data set @@ -14,118 +21,160 @@ def construct_graph(data_set): The data set as provided by cornac """ user_indices, item_indices, _ = data_set.uir_tuple - user_nodes, item_nodes = ( - torch.from_numpy(user_indices), - torch.from_numpy( - item_indices + data_set.total_users - ), # increment item node idx by num users - ) - u = torch.cat([user_nodes, item_nodes], dim=0) - v = torch.cat([item_nodes, user_nodes], dim=0) + # construct graph from the train data and add self-loops + user_selfs = [i for i in range(data_set.total_users)] + item_selfs = [i for i in range(data_set.total_items)] + + data_dict = { + (USER_KEY, "user_self", USER_KEY): (user_selfs, user_selfs), + (ITEM_KEY, "item_self", ITEM_KEY): (item_selfs, item_selfs), + (USER_KEY, "user_item", ITEM_KEY): (user_indices, item_indices), + (ITEM_KEY, "item_user", USER_KEY): (item_indices, user_indices), + } + num_dict = {USER_KEY: data_set.total_users, ITEM_KEY: data_set.total_items} - g = dgl.graph((u, v), num_nodes=(data_set.total_users + data_set.total_items)) - return g + return dgl.heterograph(data_dict, num_nodes_dict=num_dict) -class GCNLayer(nn.Module): - def __init__(self, in_size, out_size, dropout): - super(GCNLayer, self).__init__() +class NGCFLayer(nn.Module): + def __init__(self, in_size, out_size, norm_dict, dropout): + super(NGCFLayer, self).__init__() self.in_size = in_size self.out_size = out_size - - self.w1 = nn.Linear(in_size, out_size, bias=True) - self.w2 = nn.Linear(in_size, out_size, bias=True) + # weights for different types of messages + self.W1 = nn.Linear(in_size, out_size, bias=True) + self.W2 = nn.Linear(in_size, out_size, bias=True) + + # leaky relu self.leaky_relu = nn.LeakyReLU(0.2) + # dropout layer self.dropout = nn.Dropout(dropout) - torch.nn.init.xavier_uniform_(self.w1.weight) - torch.nn.init.constant_(self.w1.bias, 0) - - torch.nn.init.xavier_uniform_(self.w2.weight) - torch.nn.init.constant_(self.w2.bias, 0) - - def forward(self, graph, norm, src_embedding, dst_embedding): - with graph.local_scope(): - inner_product = torch.cat((src_embedding, dst_embedding), dim=0) - - srt, dst = graph.edges() - # out_degs = graph.out_degrees().to(src_embedding.device).float().clamp(min=1) - - msgs = self.w1(inner_product[srt]) + self.w2(inner_product[srt] * inner_product[dst]) - # norm_msgs = torch.pow(msgs, -0.5).view(-1, 1) # D^-1/2 - msgs = norm * msgs - - # inner_product = inner_product * norm_msgs - - graph.edata["h"] = msgs - # graph.ndata["h"] = inner_product - graph.update_all( - message_func=fn.copy_e("h", "m"), reduce_func=fn.sum("m", "h") - ) - - res = self.w1(inner_product) + graph.ndata["h"] - res = self.leaky_relu(res) - res = self.dropout(res) - - # in_degs = graph.in_degrees().to(src_embedding.device).float().clamp(min=1) - # norm_in_degs = torch.pow(in_degs, -0.5).view(-1, 1) # D^-1/2 - - # res = res * norm_in_degs - return res + # initialization + torch.nn.init.xavier_uniform_(self.W1.weight) + torch.nn.init.constant_(self.W1.bias, 0) + torch.nn.init.xavier_uniform_(self.W2.weight) + torch.nn.init.constant_(self.W2.bias, 0) + + # norm + self.norm_dict = norm_dict + + def forward(self, g, feat_dict): + funcs = {} # message and reduce functions dict + # for each type of edges, compute messages and reduce them all + for srctype, etype, dsttype in g.canonical_etypes: + if srctype == dsttype: # for self loops + messages = self.W1(feat_dict[srctype]) + g.nodes[srctype].data[etype] = messages # store in ndata + funcs[(srctype, etype, dsttype)] = ( + fn.copy_u(etype, "m"), + fn.sum("m", "h"), + ) # define message and reduce functions + else: + src, dst = g.edges(etype=(srctype, etype, dsttype)) + norm = self.norm_dict[(srctype, etype, dsttype)] + messages = norm * ( + self.W1(feat_dict[srctype][src]) + + self.W2(feat_dict[srctype][src] * feat_dict[dsttype][dst]) + ) # compute messages + g.edges[(srctype, etype, dsttype)].data[ + etype + ] = messages # store in edata + funcs[(srctype, etype, dsttype)] = ( + fn.copy_e(etype, "m"), + fn.sum("m", "h"), + ) # define message and reduce functions + + g.multi_update_all( + funcs, "sum" + ) # update all, reduce by first type-wisely then across different types + feature_dict = {} + for ntype in g.ntypes: + h = self.leaky_relu(g.nodes[ntype].data["h"]) # leaky relu + h = self.dropout(h) # dropout + h = F.normalize(h, dim=1, p=2) # l2 normalize + feature_dict[ntype] = h + return feature_dict class Model(nn.Module): - def __init__(self, user_size, item_size, embed_size=64, layer_size=[64, 64, 64], dropout=[0.1, 0.1, 0.1], device=None): + def __init__(self, g, in_size, layer_sizes, dropout_rates, lambda_reg, device=None): super(Model, self).__init__() - self.user_size = user_size - self.item_size = item_size - self.embedding_weights = self._init_weights(embed_size) - self.layers = nn.ModuleList([GCNLayer(embed_size, layer_size[0], dropout[0])]) - self.layers.extend( - [ - GCNLayer( - layer_size[i], layer_size[i + 1], dropout=dropout[i + 1] - ) for i in range(len(layer_size) - 1) - ] - ) + self.norm_dict = dict() + self.lambda_reg = lambda_reg self.device = device - def forward(self, graph): - user_embedding = self.embedding_weights["user_embedding"] - item_embedding = self.embedding_weights["item_embedding"] - - src, dst = graph.edges() - dst_degree = graph.in_degrees(dst).float() - src_degree = graph.out_degrees(src).float() - norm = torch.pow(src_degree * dst_degree, -0.5).view(-1, 1) # D^-1/2 - - for i, layer in enumerate(self.layers, start=1): - if i == 1: - embeddings = layer(graph, norm, user_embedding, item_embedding) - else: - embeddings = layer( - graph, norm, embeddings[: self.user_size], embeddings[self.user_size:] + for srctype, etype, dsttype in g.canonical_etypes: + src, dst = g.edges(etype=(srctype, etype, dsttype)) + dst_degree = g.in_degrees( + dst, etype=(srctype, etype, dsttype) + ).float() # obtain degrees + src_degree = g.out_degrees(src, etype=(srctype, etype, dsttype)).float() + norm = torch.pow(src_degree * dst_degree, -0.5).unsqueeze(1) # compute norm + self.norm_dict[(srctype, etype, dsttype)] = norm + + self.layers = nn.ModuleList() + self.layers.append( + NGCFLayer(in_size, layer_sizes[0], self.norm_dict, dropout_rates[0]) + ) + self.num_layers = len(layer_sizes) + for i in range(self.num_layers - 1): + self.layers.append( + NGCFLayer( + layer_sizes[i], + layer_sizes[i + 1], + self.norm_dict, + dropout_rates[i + 1], ) + ) + self.initializer = nn.init.xavier_uniform_ - user_embedding = user_embedding + embeddings[: self.user_size] - item_embedding = item_embedding + embeddings[self.user_size:] - - return user_embedding, item_embedding - - def _init_weights(self, in_size): - initializer = nn.init.xavier_uniform_ - - weights_dict = nn.ParameterDict( + # embeddings for different types of nodes + self.feature_dict = nn.ParameterDict( { - "user_embedding": nn.Parameter( - initializer(torch.empty(self.user_size, in_size)) - ), - "item_embedding": nn.Parameter( - initializer(torch.empty(self.item_size, in_size)) - ), + ntype: nn.Parameter( + self.initializer(torch.empty(g.num_nodes(ntype), in_size)) + ) + for ntype in g.ntypes } ) - return weights_dict + + def forward(self, g, users=None, pos_items=None, neg_items=None): + h_dict = {ntype: self.feature_dict[ntype] for ntype in g.ntypes} + # obtain features of each layer and concatenate them all + user_embeds = [] + item_embeds = [] + user_embeds.append(h_dict[USER_KEY]) + item_embeds.append(h_dict[ITEM_KEY]) + for layer in self.layers: + h_dict = layer(g, h_dict) + user_embeds.append(h_dict[USER_KEY]) + item_embeds.append(h_dict[ITEM_KEY]) + user_embd = torch.cat(user_embeds, 1) + item_embd = torch.cat(item_embeds, 1) + + u_g_embeddings = user_embd if users is None else user_embd[users, :] + pos_i_g_embeddings = item_embd if pos_items is None else item_embd[pos_items, :] + neg_i_g_embeddings = item_embd if neg_items is None else item_embd[neg_items, :] + + return u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings + + def loss_fn(self, users, pos_items, neg_items): + pos_scores = (users * pos_items).sum(1) + neg_scores = (users * neg_items).sum(1) + + bpr_loss = F.softplus(neg_scores - pos_scores).mean() + reg_loss = ( + (1 / 2) + * ( + torch.norm(users) ** 2 + + torch.norm(pos_items) ** 2 + + torch.norm(neg_items) ** 2 + ) + / len(users) + ) + + return bpr_loss + self.lambda_reg * reg_loss, bpr_loss, reg_loss diff --git a/cornac/models/ngcf/recom_ngcf.py b/cornac/models/ngcf/recom_ngcf.py index a2be47c58..22ead5217 100644 --- a/cornac/models/ngcf/recom_ngcf.py +++ b/cornac/models/ngcf/recom_ngcf.py @@ -29,7 +29,16 @@ class NGCF(Recommender): The name of the recommender model. num_epochs: int, default: 1000 - Maximum number of iterations or the number of epochs + Maximum number of iterations or the number of epochs. + + emb_size: int, default: 64 + Size of the node embeddings. + + layer_sizes: list, default: [64, 64, 64] + Size of the output of convolution layers. + + dropout_rates: list, defaukt: [0.1, 0.1, 0.1] + Dropout out rate for each of the convolution layers. learning_rate: float, default: 0.001 The learning rate that determines the step size at each iteration @@ -40,12 +49,6 @@ class NGCF(Recommender): test_batch_size: int, default: 100 Mini-batch size used for test set - hidden_dim: int, default: 64 - The embedding size of the model - - num_layers: int, default: 3 - Number of LightGCN Layers - early_stopping: {min_delta: float, patience: int}, optional, default: None If `None`, no early stopping. Meaning of the arguments: @@ -72,20 +75,19 @@ class NGCF(Recommender): References ---------- - * He, X., Deng, K., Wang, X., Li, Y., Zhang, Y., & Wang, M. (2020). - LightGCN: Simplifying and Powering Graph Convolution Network for - Recommendation. + * Wang, Xiang, et al. "Neural graph collaborative filtering." Proceedings of the 42nd international ACM SIGIR conference on Research and development in Information Retrieval. 2019. """ def __init__( self, name="NGCF", + emb_size=64, + layer_sizes=[64, 64, 64], + dropout_rates=[0.1, 0.1, 0.1], num_epochs=1000, learning_rate=0.001, train_batch_size=1024, test_batch_size=100, - hidden_dim=64, - num_layers=3, early_stopping=None, lambda_reg=1e-4, trainable=True, @@ -93,11 +95,11 @@ def __init__( seed=2020, ): super().__init__(name=name, trainable=trainable, verbose=verbose) - + self.emb_size = emb_size + self.layer_sizes = layer_sizes + self.dropout_rates = dropout_rates self.num_epochs = num_epochs self.learning_rate = learning_rate - self.hidden_dim = hidden_dim - self.num_layers = num_layers self.train_batch_size = train_batch_size self.test_batch_size = test_batch_size self.early_stopping = early_stopping @@ -135,19 +137,16 @@ def fit(self, train_set, val_set=None): if torch.cuda.is_available(): torch.cuda.manual_seed_all(self.seed) + graph = construct_graph(train_set).to(self.device) model = Model( - train_set.total_users, - train_set.total_items, - # self.hidden_dim, - # self.num_layers, + graph, + self.emb_size, + self.layer_sizes, + self.dropout_rates, + self.lambda_reg, ).to(self.device) - graph = construct_graph(train_set).to(self.device) - - optimizer = torch.optim.Adam( - model.parameters(), lr=self.learning_rate, weight_decay=self.lambda_reg - ) - loss_fn = torch.nn.BCELoss(reduction="sum") + optimizer = torch.optim.Adam(model.parameters(), lr=self.learning_rate) # model training pbar = trange( @@ -172,26 +171,17 @@ def fit(self, train_set, val_set=None): position=1, disable=not self.verbose, ): - user_embeddings, item_embeddings = model(graph) - - batch_u = torch.from_numpy(batch_u).long().to(self.device) - batch_i = torch.from_numpy(batch_i).long().to(self.device) - batch_j = torch.from_numpy(batch_j).long().to(self.device) - - user_embed = user_embeddings[batch_u] - positive_item_embed = item_embeddings[batch_i] - negative_item_embed = item_embeddings[batch_j] - - ui_scores = (user_embed * positive_item_embed).sum(dim=1) - uj_scores = (user_embed * negative_item_embed).sum(dim=1) + u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings = model( + graph, batch_u, batch_i, batch_j + ) - loss = loss_fn( - torch.sigmoid(ui_scores - uj_scores), torch.ones_like(ui_scores) + batch_loss, batch_bpr_loss, batch_reg_loss = model.loss_fn( + u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings ) - accum_loss += loss.cpu().item() + accum_loss += batch_loss.cpu().item() * len(batch_u) optimizer.zero_grad() - loss.backward() + batch_loss.backward() optimizer.step() accum_loss /= len(train_set.uir_tuple[0]) # normalize over all observations @@ -199,17 +189,16 @@ def fit(self, train_set, val_set=None): # store user and item embedding matrices for prediction model.eval() - self.U, self.V = model(graph) + u_embs, i_embs, _ = model(graph) + # we will use numpy for faster prediction in the score function, no need torch + self.U = u_embs.cpu().detach().numpy() + self.V = i_embs.cpu().detach().numpy() if self.early_stopping is not None and self.early_stop( **self.early_stopping ): break - # we will use numpy for faster prediction in the score function, no need torch - self.U = self.U.cpu().detach().numpy() - self.V = self.V.cpu().detach().numpy() - def monitor_value(self): """Calculating monitored value used for early stopping on validation set (`val_set`). This function will be called by `early_stop()` function. @@ -223,38 +212,17 @@ def monitor_value(self): if self.val_set is None: return None - import torch + from ...metrics import Recall + from ...eval_methods import ranking_eval - loss_fn = torch.nn.BCELoss(reduction="sum") - accum_loss = 0.0 - pbar = tqdm( - self.val_set.uij_iter(batch_size=self.test_batch_size), - desc="Validation", - total=self.val_set.num_batches(self.test_batch_size), - leave=False, - position=1, - disable=not self.verbose, - ) - for batch_u, batch_i, batch_j in pbar: - batch_u = torch.from_numpy(batch_u).long().to(self.device) - batch_i = torch.from_numpy(batch_i).long().to(self.device) - batch_j = torch.from_numpy(batch_j).long().to(self.device) - - user_embed = self.U[batch_u] - positive_item_embed = self.V[batch_i] - negative_item_embed = self.V[batch_j] - - ui_scores = (user_embed * positive_item_embed).sum(dim=1) - uj_scores = (user_embed * negative_item_embed).sum(dim=1) - - loss = loss_fn( - torch.sigmoid(ui_scores - uj_scores), torch.ones_like(ui_scores) - ) - accum_loss += loss.cpu().item() - pbar.set_postfix(val_loss=accum_loss) - - accum_loss /= len(self.val_set.uir_tuple[0]) - return -accum_loss # higher is better -> smaller loss is better + recall_20 = ranking_eval( + model=self, + metrics=[Recall(k=20)], + train_set=self.train_set, + test_set=self.val_set, + )[0][0] + + return recall_20 # Section 4.2.3 in the paper def score(self, user_idx, item_idx=None): """Predict the scores/ratings of a user for an item. diff --git a/examples/ngcf_example.py b/examples/ngcf_example.py index ebc3cbc81..434026663 100644 --- a/examples/ngcf_example.py +++ b/examples/ngcf_example.py @@ -33,31 +33,18 @@ rating_threshold=0.5, ) -lightgcn = cornac.models.LightGCN( - seed=123, - num_epochs=2000, - num_layers=3, - early_stopping={"min_delta": 1e-4, "patience": 3}, - train_batch_size=256, - learning_rate=0.001, - lambda_reg=1e-4, - verbose=True -) - # Instantiate the NGCF model ngcf = cornac.models.NGCF( seed=123, - num_epochs=2000, - num_layers=3, - early_stopping={"min_delta": 1e-4, "patience": 3}, + num_epochs=1000, + emb_size=64, + layer_sizes=[64, 64, 64], + dropout_rates=[0.1, 0.1, 0.1], + early_stopping={"min_delta": 1e-4, "patience": 50}, train_batch_size=256, learning_rate=0.001, - lambda_reg=1e-4, - verbose=True -) - -gcmc = cornac.models.GCMC( - seed=123, + lambda_reg=1e-5, + verbose=True, ) # Instantiate evaluation measures @@ -67,7 +54,7 @@ # Put everything together into an experiment and run it cornac.Experiment( eval_method=ratio_split, - models=[lightgcn, ngcf], + models=[ngcf], metrics=[rec_20, ndcg_20], user_based=True, ).run() From 1c446093a0e31d624eedb5e8c34b15e11f259795 Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Wed, 11 Oct 2023 18:25:17 +0800 Subject: [PATCH 06/23] added sanity check --- cornac/models/ngcf/ngcf.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cornac/models/ngcf/ngcf.py b/cornac/models/ngcf/ngcf.py index 6d25d47eb..64e473bbc 100644 --- a/cornac/models/ngcf/ngcf.py +++ b/cornac/models/ngcf/ngcf.py @@ -117,6 +117,11 @@ def __init__(self, g, in_size, layer_sizes, dropout_rates, lambda_reg, device=No self.norm_dict[(srctype, etype, dsttype)] = norm self.layers = nn.ModuleList() + + # sanity check, just to ensure layer sizes and dropout_rates have the same size + assert len(layer_sizes) == len(dropout_rates), "'layer_sizes' and " \ + "'dropout_rates' must be of the same size" + self.layers.append( NGCFLayer(in_size, layer_sizes[0], self.norm_dict, dropout_rates[0]) ) From 60c164140a815fa38c133ec96b1a0e5ee3a285bd Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Wed, 11 Oct 2023 18:28:30 +0800 Subject: [PATCH 07/23] Changed train batch size in example to 1024 --- examples/ngcf_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ngcf_example.py b/examples/ngcf_example.py index 434026663..89abba836 100644 --- a/examples/ngcf_example.py +++ b/examples/ngcf_example.py @@ -41,7 +41,7 @@ layer_sizes=[64, 64, 64], dropout_rates=[0.1, 0.1, 0.1], early_stopping={"min_delta": 1e-4, "patience": 50}, - train_batch_size=256, + train_batch_size=1024, learning_rate=0.001, lambda_reg=1e-5, verbose=True, From e604a486795117b2d965d711a9b3e870391b944a Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Thu, 12 Oct 2023 11:47:02 +0800 Subject: [PATCH 08/23] Updated readme for example folder --- examples/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/README.md b/examples/README.md index 966f6c6ba..5ae6d09d5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -34,6 +34,8 @@ [mcf_office.py](mcf_office.py) - Fit Matrix Co-Factorization (MCF) to the Amazon Office dataset. +[ngcf_example.py](ngcf_example.py) - NGCF example with CiteULike dataset. + [pcrl_example.py](pcrl_example.py) - Probabilistic Collaborative Representation Learning (PCRL) Amazon Office dataset. [sbpr_epinions.py](sbpr_epinions.py) - Social Bayesian Personalized Ranking (SBPR) with Epinions dataset. From c3b23f558631eaaf6d6483dd10cc6cdd7e933a19 Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Thu, 12 Oct 2023 11:49:43 +0800 Subject: [PATCH 09/23] Update Readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 626f3c4c9..7ecc17ae8 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ The recommender models supported by Cornac are listed below. Why don't you join | | [Hybrid neural recommendation with joint deep representation learning of ratings and reviews (HRDR)](cornac/models/hrdr), [paper](https://www.sciencedirect.com/science/article/abs/pii/S0925231219313207) | [requirements.txt](cornac/models/hrdr/requirements.txt) | [hrdr_example.py](examples/hrdr_example.py) | | [LightGCN: Simplifying and Powering Graph Convolution Network for Recommendation](cornac/models/lightgcn), [paper](https://arxiv.org/pdf/2002.02126.pdf) | [requirements.txt](cornac/models/lightgcn/requirements.txt) | [lightgcn_example.py](examples/lightgcn_example.py) | 2019 | [Embarrassingly Shallow Autoencoders for Sparse Data (EASEá´¿)](cornac/models/ease), [paper](https://arxiv.org/pdf/1905.03375.pdf) | N/A | [ease_movielens.py](examples/ease_movielens.py) +| | [Neural Graph Collaborative Filtering](cornac/models/ngcf), [paper](https://arxiv.org/pdf/1905.08108.pdf) | [requirements.txt](cornac/models/ngcf/requirements.txt) | [ngcf_example.py](examples/ngcf_example.py) | 2018 | [Collaborative Context Poisson Factorization (C2PF)](cornac/models/c2pf), [paper](https://www.ijcai.org/proceedings/2018/0370.pdf) | N/A | [c2pf_exp.py](examples/c2pf_example.py) | | [Graph Convolutional Matrix Completion (GCMC)](cornac/models/gcmc), [paper](https://www.kdd.org/kdd2018/files/deep-learning-day/DLDay18_paper_32.pdf) | [requirements.txt](cornac/models/gcmc/requirements.txt) | [gcmc_example.py](examples/gcmc_example.py) | | [Multi-Task Explainable Recommendation (MTER)](cornac/models/mter), [paper](https://arxiv.org/pdf/1806.03568.pdf) | N/A | [mter_exp.py](examples/mter_example.py) From b77a2684b655a3e3e38a0470cdf7bbec9bc43704 Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Thu, 12 Oct 2023 11:57:57 +0800 Subject: [PATCH 10/23] update docs --- docs/source/models.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/models.rst b/docs/source/models.rst index f36d616a0..ba603a80b 100644 --- a/docs/source/models.rst +++ b/docs/source/models.rst @@ -68,6 +68,11 @@ Neural Attention Rating Regression with Review-level Explanations (NARRE) .. automodule:: cornac.models.narre.recom_narre :members: +Neural Graph Collaborative Filtering (NGCF) +---------------------------------------------------- +.. automodule:: cornac.models.ngcf.recom_ngcf + :members: + Probabilistic Collaborative Representation Learning (PCRL) ------------------------------------------------------------ .. automodule:: cornac.models.pcrl.recom_pcrl From 05918476970a628e696fe153ea7ca68b6985bd40 Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Thu, 12 Oct 2023 12:21:32 +0800 Subject: [PATCH 11/23] Update block comment --- cornac/models/ngcf/recom_ngcf.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cornac/models/ngcf/recom_ngcf.py b/cornac/models/ngcf/recom_ngcf.py index 22ead5217..7284b8af7 100644 --- a/cornac/models/ngcf/recom_ngcf.py +++ b/cornac/models/ngcf/recom_ngcf.py @@ -21,24 +21,25 @@ class NGCF(Recommender): """ - NGCF + Neural Graph Collaborative Filtering Parameters ---------- name: string, default: 'NGCF' The name of the recommender model. - num_epochs: int, default: 1000 - Maximum number of iterations or the number of epochs. - emb_size: int, default: 64 Size of the node embeddings. layer_sizes: list, default: [64, 64, 64] Size of the output of convolution layers. - dropout_rates: list, defaukt: [0.1, 0.1, 0.1] - Dropout out rate for each of the convolution layers. + dropout_rates: list, default: [0.1, 0.1, 0.1] + Dropout rate for each of the convolution layers. + - Number of values should be the same as 'layer_sizes' + + num_epochs: int, default: 1000 + Maximum number of iterations or the number of epochs. learning_rate: float, default: 0.001 The learning rate that determines the step size at each iteration From fa0f29e0327dd04258594e318ee6950a0a03177f Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Thu, 12 Oct 2023 19:04:18 +0800 Subject: [PATCH 12/23] WIP --- cornac/models/lightgcn2/__init__.py | 16 ++ cornac/models/lightgcn2/lightgcn.py | 129 +++++++++++ cornac/models/lightgcn2/recom_lightgcn.py | 253 ++++++++++++++++++++++ cornac/models/lightgcn2/requirements.txt | 2 + 4 files changed, 400 insertions(+) create mode 100644 cornac/models/lightgcn2/__init__.py create mode 100644 cornac/models/lightgcn2/lightgcn.py create mode 100644 cornac/models/lightgcn2/recom_lightgcn.py create mode 100644 cornac/models/lightgcn2/requirements.txt diff --git a/cornac/models/lightgcn2/__init__.py b/cornac/models/lightgcn2/__init__.py new file mode 100644 index 000000000..0d239a783 --- /dev/null +++ b/cornac/models/lightgcn2/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2018 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from .recom_lightgcn import LightGCN diff --git a/cornac/models/lightgcn2/lightgcn.py b/cornac/models/lightgcn2/lightgcn.py new file mode 100644 index 000000000..15cae5010 --- /dev/null +++ b/cornac/models/lightgcn2/lightgcn.py @@ -0,0 +1,129 @@ +# Reference: https://github.com/dmlc/dgl/blob/master/examples/pytorch/NGCF/NGCF/model.py + +import torch +import torch.nn as nn +import torch.nn.functional as F +import dgl +import dgl.function as fn + + +USER_KEY = "user" +ITEM_KEY = "item" + + +def construct_graph(data_set): + """ + Generates graph given a cornac data set + + Parameters + ---------- + data_set : cornac.data.dataset.Dataset + The data set as provided by cornac + """ + user_indices, item_indices, _ = data_set.uir_tuple + + data_dict = { + (USER_KEY, "user_item", ITEM_KEY): (user_indices, item_indices), + (ITEM_KEY, "item_user", USER_KEY): (item_indices, user_indices), + } + num_dict = {USER_KEY: data_set.total_users, ITEM_KEY: data_set.total_items} + + return dgl.heterograph(data_dict, num_nodes_dict=num_dict) + + +class GCNLayer(nn.Module): + def __init__(self, norm_dict): + super(GCNLayer, self).__init__() + + # norm + self.norm_dict = norm_dict + + def forward(self, g, feat_dict): + funcs = {} # message and reduce functions dict + # for each type of edges, compute messages and reduce them all + for srctype, etype, dsttype in g.canonical_etypes: + src, dst = g.edges(etype=(srctype, etype, dsttype)) + norm = self.norm_dict[(srctype, etype, dsttype)] + # TODO: CHECK HERE + messages = norm * feat_dict[srctype][src] * feat_dict[dsttype][dst] # compute messages + g.edges[(srctype, etype, dsttype)].data[ + etype + ] = messages # store in edata + funcs[(srctype, etype, dsttype)] = ( + fn.copy_e(etype, "m"), + fn.sum("m", "h"), + ) # define message and reduce functions + + g.multi_update_all( + funcs, "sum" + ) # update all, reduce by first type-wisely then across different types + feature_dict = {} + for ntype in g.ntypes: + h = F.normalize(g.nodes[ntype].data["h"], dim=1, p=2) # l2 normalize + feature_dict[ntype] = h + return feature_dict + + +class Model(nn.Module): + def __init__(self, g, in_size, num_layers, lambda_reg, device=None): + super(Model, self).__init__() + self.norm_dict = dict() + self.lambda_reg = lambda_reg + self.device = device + + for srctype, etype, dsttype in g.canonical_etypes: + src, dst = g.edges(etype=(srctype, etype, dsttype)) + dst_degree = g.in_degrees( + dst, etype=(srctype, etype, dsttype) + ).float() # obtain degrees + src_degree = g.out_degrees(src, etype=(srctype, etype, dsttype)).float() + norm = torch.pow(src_degree * dst_degree, -0.5).unsqueeze(1) # compute norm + self.norm_dict[(srctype, etype, dsttype)] = norm + + self.layers = nn.ModuleList([GCNLayer(self.norm_dict) for _ in range(num_layers)]) + + self.initializer = nn.init.xavier_uniform_ + + # embeddings for different types of nodes + self.feature_dict = nn.ParameterDict( + { + ntype: nn.Parameter( + self.initializer(torch.empty(g.num_nodes(ntype), in_size)) + ) + for ntype in g.ntypes + } + ) + + def forward(self, g, users=None, pos_items=None, neg_items=None): + h_dict = {ntype: self.feature_dict[ntype] for ntype in g.ntypes} + # obtain features of each layer and concatenate them all + user_embeds = h_dict[USER_KEY] + item_embeds = h_dict[ITEM_KEY] + + for i, layer in enumerate(self.layers): + h_dict = layer(g, h_dict) + user_embeds = user_embeds + (h_dict[USER_KEY] * 1 / (i + 1)) + item_embeds = item_embeds + (h_dict[ITEM_KEY] * 1 / (i + 1)) + + u_g_embeddings = user_embeds if users is None else user_embeds[users, :] + pos_i_g_embeddings = item_embeds if pos_items is None else item_embeds[pos_items, :] + neg_i_g_embeddings = item_embeds if neg_items is None else item_embeds[neg_items, :] + + return u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings + + def loss_fn(self, users, pos_items, neg_items): + pos_scores = (users * pos_items).sum(1) + neg_scores = (users * neg_items).sum(1) + + bpr_loss = F.softplus(neg_scores - pos_scores).mean() + reg_loss = ( + (1 / 2) + * ( + torch.norm(users) ** 2 + + torch.norm(pos_items) ** 2 + + torch.norm(neg_items) ** 2 + ) + / len(users) + ) + + return bpr_loss + self.lambda_reg * reg_loss, bpr_loss, reg_loss diff --git a/cornac/models/lightgcn2/recom_lightgcn.py b/cornac/models/lightgcn2/recom_lightgcn.py new file mode 100644 index 000000000..0b56dbf9c --- /dev/null +++ b/cornac/models/lightgcn2/recom_lightgcn.py @@ -0,0 +1,253 @@ +# Copyright 2018 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from ..recommender import Recommender +from ...exception import ScoreException + +from tqdm.auto import tqdm, trange + + +class LightGCN(Recommender): + """ + LightGCN + + Parameters + ---------- + name: string, default: 'LightGCN' + The name of the recommender model. + + emb_size: int, default: 64 + Size of the node embeddings. + + num_epochs: int, default: 1000 + Maximum number of iterations or the number of epochs. + + learning_rate: float, default: 0.001 + The learning rate that determines the step size at each iteration + + train_batch_size: int, default: 1024 + Mini-batch size used for train set + + test_batch_size: int, default: 100 + Mini-batch size used for test set + + early_stopping: {min_delta: float, patience: int}, optional, default: None + If `None`, no early stopping. Meaning of the arguments: + + - `min_delta`: the minimum increase in monitored value on validation + set to be considered as improvement, + i.e. an increment of less than min_delta will count as + no improvement. + + - `patience`: number of epochs with no improvement after which + training should be stopped. + + lambda_reg: float, default: 1e-4 + Weight decay for the L2 normalization + + trainable: boolean, optional, default: True + When False, the model is not trained and Cornac assumes that the model + is already pre-trained. + + verbose: boolean, optional, default: False + When True, some running logs are displayed. + + seed: int, optional, default: 2020 + Random seed for parameters initialization. + + References + ---------- + * Wang, Xiang, et al. "Neural graph collaborative filtering." Proceedings of the 42nd international ACM SIGIR conference on Research and development in Information Retrieval. 2019. + """ + + def __init__( + self, + name="LightGCN", + emb_size=64, + num_epochs=1000, + learning_rate=0.001, + train_batch_size=1024, + test_batch_size=100, + num_layers=3, + early_stopping=None, + lambda_reg=1e-4, + trainable=True, + verbose=False, + seed=2020, + ): + super().__init__(name=name, trainable=trainable, verbose=verbose) + self.emb_size = emb_size + self.num_epochs = num_epochs + self.learning_rate = learning_rate + self.train_batch_size = train_batch_size + self.test_batch_size = test_batch_size + self.num_layers = num_layers + self.early_stopping = early_stopping + self.lambda_reg = lambda_reg + self.seed = seed + + def fit(self, train_set, val_set=None): + """Fit the model to observations. + + Parameters + ---------- + train_set: :obj:`cornac.data.Dataset`, required + User-Item preference data as well as additional modalities. + + val_set: :obj:`cornac.data.Dataset`, optional, default: None + User-Item preference data for model selection purposes (e.g., early stopping). + + Returns + ------- + self : object + """ + Recommender.fit(self, train_set, val_set) + + if not self.trainable: + return self + + # model setup + import torch + from .lightgcn import Model + from .lightgcn import construct_graph + + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + if self.seed is not None: + torch.manual_seed(self.seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(self.seed) + + graph = construct_graph(train_set).to(self.device) + model = Model( + graph, + self.emb_size, + self.num_layers, + self.lambda_reg, + ).to(self.device) + + optimizer = torch.optim.Adam(model.parameters(), lr=self.learning_rate) + + # model training + pbar = trange( + self.num_epochs, + desc="Training", + unit="iter", + position=0, + leave=False, + disable=not self.verbose, + ) + for _ in pbar: + model.train() + accum_loss = 0.0 + for batch_u, batch_i, batch_j in tqdm( + train_set.uij_iter( + batch_size=self.train_batch_size, + shuffle=True, + ), + desc="Epoch", + total=train_set.num_batches(self.train_batch_size), + leave=False, + position=1, + disable=not self.verbose, + ): + u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings = model( + graph, batch_u, batch_i, batch_j + ) + + batch_loss, batch_bpr_loss, batch_reg_loss = model.loss_fn( + u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings + ) + accum_loss += batch_loss.cpu().item() * len(batch_u) + + optimizer.zero_grad() + batch_loss.backward() + optimizer.step() + + accum_loss /= len(train_set.uir_tuple[0]) # normalize over all observations + pbar.set_postfix(loss=accum_loss) + + # store user and item embedding matrices for prediction + model.eval() + u_embs, i_embs, _ = model(graph) + # we will use numpy for faster prediction in the score function, no need torch + self.U = u_embs.cpu().detach().numpy() + self.V = i_embs.cpu().detach().numpy() + + if self.early_stopping is not None and self.early_stop( + **self.early_stopping + ): + break + + def monitor_value(self): + """Calculating monitored value used for early stopping on validation set (`val_set`). + This function will be called by `early_stop()` function. + + Returns + ------- + res : float + Monitored value on validation set. + Return `None` if `val_set` is `None`. + """ + # TODO: Change this to BPR loss! + if self.val_set is None: + return None + + from ...metrics import Recall + from ...eval_methods import ranking_eval + + recall_20 = ranking_eval( + model=self, + metrics=[Recall(k=20)], + train_set=self.train_set, + test_set=self.val_set, + verbose=True + )[0][0] + + return recall_20 # Section 4.2.3 in the paper + + def score(self, user_idx, item_idx=None): + """Predict the scores/ratings of a user for an item. + + Parameters + ---------- + user_idx: int, required + The index of the user for whom to perform score prediction. + + item_idx: int, optional, default: None + The index of the item for which to perform score prediction. + If None, scores for all known items will be returned. + + Returns + ------- + res : A scalar or a Numpy array + Relative scores that the user gives to the item or to all known items + + """ + if item_idx is None: + if self.train_set.is_unk_user(user_idx): + raise ScoreException( + "Can't make score prediction for (user_id=%d)" % user_idx + ) + known_item_scores = self.V.dot(self.U[user_idx, :]) + return known_item_scores + else: + if self.train_set.is_unk_user(user_idx) or self.train_set.is_unk_item( + item_idx + ): + raise ScoreException( + "Can't make score prediction for (user_id=%d, item_id=%d)" + % (user_idx, item_idx) + ) + return self.V[item_idx, :].dot(self.U[user_idx, :]) diff --git a/cornac/models/lightgcn2/requirements.txt b/cornac/models/lightgcn2/requirements.txt new file mode 100644 index 000000000..32f294fbc --- /dev/null +++ b/cornac/models/lightgcn2/requirements.txt @@ -0,0 +1,2 @@ +torch>=2.0.0 +dgl>=1.1.0 \ No newline at end of file From 8797bc5c95a1fbc34d0c53020ecdf313380a6d5d Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Fri, 13 Oct 2023 16:51:24 +0800 Subject: [PATCH 13/23] Updated validation metric --- cornac/models/lightgcn2/recom_lightgcn.py | 49 ++++++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/cornac/models/lightgcn2/recom_lightgcn.py b/cornac/models/lightgcn2/recom_lightgcn.py index 0b56dbf9c..d08653a43 100644 --- a/cornac/models/lightgcn2/recom_lightgcn.py +++ b/cornac/models/lightgcn2/recom_lightgcn.py @@ -200,22 +200,49 @@ def monitor_value(self): Monitored value on validation set. Return `None` if `val_set` is `None`. """ - # TODO: Change this to BPR loss! if self.val_set is None: return None - from ...metrics import Recall - from ...eval_methods import ranking_eval + import torch + import torch.nn.functional as F + + accum_loss = 0.0 + pbar = tqdm( + self.val_set.uij_iter(batch_size=self.test_batch_size), + desc="Validation", + total=self.val_set.num_batches(self.test_batch_size), + leave=False, + position=1, + disable=not self.verbose, + ) + for batch_u, batch_i, batch_j in pbar: + batch_u = torch.from_numpy(batch_u).long() + batch_i = torch.from_numpy(batch_i).long() + batch_j = torch.from_numpy(batch_j).long() + + u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings = self.U[batch_u], self.V[batch_i], self.V[batch_j] + u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings = torch.from_numpy(u_g_embeddings), torch.from_numpy(pos_i_g_embeddings), torch.from_numpy(neg_i_g_embeddings) + + pos_scores = (u_g_embeddings * pos_i_g_embeddings).sum(1) + neg_scores = (u_g_embeddings * neg_i_g_embeddings).sum(1) + + bpr_loss = F.softplus(neg_scores - pos_scores).mean() + reg_loss = ( + (1 / 2) + * ( + torch.norm(u_g_embeddings) ** 2 + + torch.norm(pos_i_g_embeddings) ** 2 + + torch.norm(neg_i_g_embeddings) ** 2 + ) + / len(u_g_embeddings) + ) - recall_20 = ranking_eval( - model=self, - metrics=[Recall(k=20)], - train_set=self.train_set, - test_set=self.val_set, - verbose=True - )[0][0] + batch_loss = bpr_loss + self.lambda_reg * reg_loss + accum_loss += batch_loss.cpu().item() * len(batch_u) + pbar.set_postfix(val_loss=accum_loss) - return recall_20 # Section 4.2.3 in the paper + accum_loss /= len(self.val_set.uir_tuple[0]) # normalize over all observations + return -accum_loss # higher is better -> smaller loss is better def score(self, user_idx, item_idx=None): """Predict the scores/ratings of a user for an item. From 1ae4584e5b2e74d1e9ab7f64bf468373e5c7d960 Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Fri, 13 Oct 2023 16:51:43 +0800 Subject: [PATCH 14/23] Updated message handling --- cornac/models/lightgcn2/lightgcn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cornac/models/lightgcn2/lightgcn.py b/cornac/models/lightgcn2/lightgcn.py index 15cae5010..cb9ea1bc3 100644 --- a/cornac/models/lightgcn2/lightgcn.py +++ b/cornac/models/lightgcn2/lightgcn.py @@ -45,7 +45,7 @@ def forward(self, g, feat_dict): src, dst = g.edges(etype=(srctype, etype, dsttype)) norm = self.norm_dict[(srctype, etype, dsttype)] # TODO: CHECK HERE - messages = norm * feat_dict[srctype][src] * feat_dict[dsttype][dst] # compute messages + messages = norm * feat_dict[srctype][src] # compute messages g.edges[(srctype, etype, dsttype)].data[ etype ] = messages # store in edata From ea88208bda887740a1dd96452b457dc5f82b1670 Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Fri, 13 Oct 2023 16:53:00 +0800 Subject: [PATCH 15/23] Added legacy lightgcn for comparison purposes --- cornac/models/__init__.py | 3 ++- cornac/models/lightgcn/__init__.py | 2 +- cornac/models/lightgcn/recom_lightgcn.py | 2 +- examples/lightgcn_example.py | 14 +++++++++++++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cornac/models/__init__.py b/cornac/models/__init__.py index 82b2a9e0d..094bf9a8d 100644 --- a/cornac/models/__init__.py +++ b/cornac/models/__init__.py @@ -41,7 +41,8 @@ from .ibpr import IBPR from .knn import ItemKNN from .knn import UserKNN -from .lightgcn import LightGCN +from .lightgcn import LightGCN1 +from .lightgcn2 import LightGCN from .mcf import MCF from .mf import MF from .mmmf import MMMF diff --git a/cornac/models/lightgcn/__init__.py b/cornac/models/lightgcn/__init__.py index 0d239a783..80d36255e 100644 --- a/cornac/models/lightgcn/__init__.py +++ b/cornac/models/lightgcn/__init__.py @@ -13,4 +13,4 @@ # limitations under the License. # ============================================================================ -from .recom_lightgcn import LightGCN +from .recom_lightgcn import LightGCN1 diff --git a/cornac/models/lightgcn/recom_lightgcn.py b/cornac/models/lightgcn/recom_lightgcn.py index 669b4ea62..a389b295a 100644 --- a/cornac/models/lightgcn/recom_lightgcn.py +++ b/cornac/models/lightgcn/recom_lightgcn.py @@ -19,7 +19,7 @@ from tqdm.auto import tqdm, trange -class LightGCN(Recommender): +class LightGCN1(Recommender): """ LightGCN diff --git a/examples/lightgcn_example.py b/examples/lightgcn_example.py index 11efb6869..004130fd4 100644 --- a/examples/lightgcn_example.py +++ b/examples/lightgcn_example.py @@ -33,6 +33,18 @@ rating_threshold=0.5, ) +lightgcn1 = cornac.models.LightGCN1( + name="LightGCN-old", + seed=123, + num_epochs=2000, + num_layers=3, + early_stopping={"min_delta": 1e-4, "patience": 3}, + train_batch_size=256, + learning_rate=0.001, + lambda_reg=1e-4, + verbose=True +) + # Instantiate the LightGCN model lightgcn = cornac.models.LightGCN( seed=123, @@ -52,7 +64,7 @@ # Put everything together into an experiment and run it cornac.Experiment( eval_method=ratio_split, - models=[lightgcn], + models=[lightgcn1, lightgcn], metrics=[rec_20, ndcg_20], user_based=True, ).run() From 9b45f88683714d67b4218c1eeaef22f65b746a21 Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Sat, 14 Oct 2023 01:36:15 +0800 Subject: [PATCH 16/23] Changed to follow 'a_k = 1/(k+1)', k instead of i --- cornac/models/lightgcn2/lightgcn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cornac/models/lightgcn2/lightgcn.py b/cornac/models/lightgcn2/lightgcn.py index cb9ea1bc3..47172301c 100644 --- a/cornac/models/lightgcn2/lightgcn.py +++ b/cornac/models/lightgcn2/lightgcn.py @@ -100,10 +100,10 @@ def forward(self, g, users=None, pos_items=None, neg_items=None): user_embeds = h_dict[USER_KEY] item_embeds = h_dict[ITEM_KEY] - for i, layer in enumerate(self.layers): + for k, layer in enumerate(self.layers): h_dict = layer(g, h_dict) - user_embeds = user_embeds + (h_dict[USER_KEY] * 1 / (i + 1)) - item_embeds = item_embeds + (h_dict[ITEM_KEY] * 1 / (i + 1)) + user_embeds = user_embeds + (h_dict[USER_KEY] * 1 / (k + 1)) + item_embeds = item_embeds + (h_dict[ITEM_KEY] * 1 / (k + 1)) u_g_embeddings = user_embeds if users is None else user_embeds[users, :] pos_i_g_embeddings = item_embeds if pos_items is None else item_embeds[pos_items, :] From e1bfec0e1657e3d127e20774b19a6249ea8af4be Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Sat, 14 Oct 2023 01:39:20 +0800 Subject: [PATCH 17/23] Changed early stopping technique to follow NGCF --- cornac/models/lightgcn2/recom_lightgcn.py | 52 ++++++----------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/cornac/models/lightgcn2/recom_lightgcn.py b/cornac/models/lightgcn2/recom_lightgcn.py index d08653a43..70554062b 100644 --- a/cornac/models/lightgcn2/recom_lightgcn.py +++ b/cornac/models/lightgcn2/recom_lightgcn.py @@ -203,46 +203,18 @@ def monitor_value(self): if self.val_set is None: return None - import torch - import torch.nn.functional as F - - accum_loss = 0.0 - pbar = tqdm( - self.val_set.uij_iter(batch_size=self.test_batch_size), - desc="Validation", - total=self.val_set.num_batches(self.test_batch_size), - leave=False, - position=1, - disable=not self.verbose, - ) - for batch_u, batch_i, batch_j in pbar: - batch_u = torch.from_numpy(batch_u).long() - batch_i = torch.from_numpy(batch_i).long() - batch_j = torch.from_numpy(batch_j).long() - - u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings = self.U[batch_u], self.V[batch_i], self.V[batch_j] - u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings = torch.from_numpy(u_g_embeddings), torch.from_numpy(pos_i_g_embeddings), torch.from_numpy(neg_i_g_embeddings) - - pos_scores = (u_g_embeddings * pos_i_g_embeddings).sum(1) - neg_scores = (u_g_embeddings * neg_i_g_embeddings).sum(1) - - bpr_loss = F.softplus(neg_scores - pos_scores).mean() - reg_loss = ( - (1 / 2) - * ( - torch.norm(u_g_embeddings) ** 2 - + torch.norm(pos_i_g_embeddings) ** 2 - + torch.norm(neg_i_g_embeddings) ** 2 - ) - / len(u_g_embeddings) - ) - - batch_loss = bpr_loss + self.lambda_reg * reg_loss - accum_loss += batch_loss.cpu().item() * len(batch_u) - pbar.set_postfix(val_loss=accum_loss) - - accum_loss /= len(self.val_set.uir_tuple[0]) # normalize over all observations - return -accum_loss # higher is better -> smaller loss is better + from ...metrics import Recall + from ...eval_methods import ranking_eval + + recall_20 = ranking_eval( + model=self, + metrics=[Recall(k=20)], + train_set=self.train_set, + test_set=self.val_set, + verbose=True + )[0][0] + + return recall_20 # Section 4.1.2 in the paper, same strategy as NGCF. def score(self, user_idx, item_idx=None): """Predict the scores/ratings of a user for an item. From 09457941e37167f54e4acdf679256dd957f307c0 Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Sat, 14 Oct 2023 01:45:43 +0800 Subject: [PATCH 18/23] remove test_batchsize, early stop verbose to false --- cornac/models/lightgcn2/recom_lightgcn.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cornac/models/lightgcn2/recom_lightgcn.py b/cornac/models/lightgcn2/recom_lightgcn.py index 70554062b..effd015f2 100644 --- a/cornac/models/lightgcn2/recom_lightgcn.py +++ b/cornac/models/lightgcn2/recom_lightgcn.py @@ -40,9 +40,6 @@ class LightGCN(Recommender): train_batch_size: int, default: 1024 Mini-batch size used for train set - test_batch_size: int, default: 100 - Mini-batch size used for test set - early_stopping: {min_delta: float, patience: int}, optional, default: None If `None`, no early stopping. Meaning of the arguments: @@ -79,7 +76,6 @@ def __init__( num_epochs=1000, learning_rate=0.001, train_batch_size=1024, - test_batch_size=100, num_layers=3, early_stopping=None, lambda_reg=1e-4, @@ -92,7 +88,6 @@ def __init__( self.num_epochs = num_epochs self.learning_rate = learning_rate self.train_batch_size = train_batch_size - self.test_batch_size = test_batch_size self.num_layers = num_layers self.early_stopping = early_stopping self.lambda_reg = lambda_reg @@ -210,8 +205,7 @@ def monitor_value(self): model=self, metrics=[Recall(k=20)], train_set=self.train_set, - test_set=self.val_set, - verbose=True + test_set=self.val_set )[0][0] return recall_20 # Section 4.1.2 in the paper, same strategy as NGCF. From 8120973f624a66c591fa4a1f344953f4c10560f3 Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Sun, 15 Oct 2023 00:41:12 +0800 Subject: [PATCH 19/23] Changed parameters to align with paper and ngcf --- examples/lightgcn_example.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/examples/lightgcn_example.py b/examples/lightgcn_example.py index 004130fd4..b91660063 100644 --- a/examples/lightgcn_example.py +++ b/examples/lightgcn_example.py @@ -33,25 +33,13 @@ rating_threshold=0.5, ) -lightgcn1 = cornac.models.LightGCN1( - name="LightGCN-old", - seed=123, - num_epochs=2000, - num_layers=3, - early_stopping={"min_delta": 1e-4, "patience": 3}, - train_batch_size=256, - learning_rate=0.001, - lambda_reg=1e-4, - verbose=True -) - # Instantiate the LightGCN model lightgcn = cornac.models.LightGCN( seed=123, - num_epochs=2000, + num_epochs=1000, num_layers=3, - early_stopping={"min_delta": 1e-4, "patience": 3}, - train_batch_size=256, + early_stopping={"min_delta": 1e-4, "patience": 50}, + train_batch_size=1024, learning_rate=0.001, lambda_reg=1e-4, verbose=True @@ -64,7 +52,7 @@ # Put everything together into an experiment and run it cornac.Experiment( eval_method=ratio_split, - models=[lightgcn1, lightgcn], + models=[lightgcn], metrics=[rec_20, ndcg_20], user_based=True, ).run() From 27529c40e4144cb416931dca37f4fe04b190e3ea Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Tue, 17 Oct 2023 11:50:54 +0800 Subject: [PATCH 20/23] refractor codes --- cornac/models/lightgcn/__init__.py | 2 +- cornac/models/lightgcn/lightgcn.py | 161 ++++++++------ cornac/models/lightgcn/recom_lightgcn.py | 116 +++------- cornac/models/lightgcn2/__init__.py | 16 -- cornac/models/lightgcn2/lightgcn.py | 129 ------------ cornac/models/lightgcn2/recom_lightgcn.py | 246 ---------------------- cornac/models/lightgcn2/requirements.txt | 2 - 7 files changed, 130 insertions(+), 542 deletions(-) delete mode 100644 cornac/models/lightgcn2/__init__.py delete mode 100644 cornac/models/lightgcn2/lightgcn.py delete mode 100644 cornac/models/lightgcn2/recom_lightgcn.py delete mode 100644 cornac/models/lightgcn2/requirements.txt diff --git a/cornac/models/lightgcn/__init__.py b/cornac/models/lightgcn/__init__.py index 80d36255e..0d239a783 100644 --- a/cornac/models/lightgcn/__init__.py +++ b/cornac/models/lightgcn/__init__.py @@ -13,4 +13,4 @@ # limitations under the License. # ============================================================================ -from .recom_lightgcn import LightGCN1 +from .recom_lightgcn import LightGCN diff --git a/cornac/models/lightgcn/lightgcn.py b/cornac/models/lightgcn/lightgcn.py index 22983ab3b..47172301c 100644 --- a/cornac/models/lightgcn/lightgcn.py +++ b/cornac/models/lightgcn/lightgcn.py @@ -1,9 +1,16 @@ +# Reference: https://github.com/dmlc/dgl/blob/master/examples/pytorch/NGCF/NGCF/model.py + import torch import torch.nn as nn +import torch.nn.functional as F import dgl import dgl.function as fn +USER_KEY = "user" +ITEM_KEY = "item" + + def construct_graph(data_set): """ Generates graph given a cornac data set @@ -14,89 +21,109 @@ def construct_graph(data_set): The data set as provided by cornac """ user_indices, item_indices, _ = data_set.uir_tuple - user_nodes, item_nodes = ( - torch.from_numpy(user_indices), - torch.from_numpy( - item_indices + data_set.total_users - ), # increment item node idx by num users - ) - u = torch.cat([user_nodes, item_nodes], dim=0) - v = torch.cat([item_nodes, user_nodes], dim=0) + data_dict = { + (USER_KEY, "user_item", ITEM_KEY): (user_indices, item_indices), + (ITEM_KEY, "item_user", USER_KEY): (item_indices, user_indices), + } + num_dict = {USER_KEY: data_set.total_users, ITEM_KEY: data_set.total_items} - g = dgl.graph((u, v), num_nodes=(data_set.total_users + data_set.total_items)) - return g + return dgl.heterograph(data_dict, num_nodes_dict=num_dict) class GCNLayer(nn.Module): - def __init__(self): + def __init__(self, norm_dict): super(GCNLayer, self).__init__() - def forward(self, graph, src_embedding, dst_embedding): - with graph.local_scope(): - inner_product = torch.cat((src_embedding, dst_embedding), dim=0) - - out_degs = graph.out_degrees().to(src_embedding.device).float().clamp(min=1) - norm_out_degs = torch.pow(out_degs, -0.5).view(-1, 1) # D^-1/2 - - inner_product = inner_product * norm_out_degs - - graph.ndata["h"] = inner_product - graph.update_all( - message_func=fn.copy_u("h", "m"), reduce_func=fn.sum("m", "h") - ) - - res = graph.ndata["h"] - - in_degs = graph.in_degrees().to(src_embedding.device).float().clamp(min=1) - norm_in_degs = torch.pow(in_degs, -0.5).view(-1, 1) # D^-1/2 - - res = res * norm_in_degs - return res + # norm + self.norm_dict = norm_dict + + def forward(self, g, feat_dict): + funcs = {} # message and reduce functions dict + # for each type of edges, compute messages and reduce them all + for srctype, etype, dsttype in g.canonical_etypes: + src, dst = g.edges(etype=(srctype, etype, dsttype)) + norm = self.norm_dict[(srctype, etype, dsttype)] + # TODO: CHECK HERE + messages = norm * feat_dict[srctype][src] # compute messages + g.edges[(srctype, etype, dsttype)].data[ + etype + ] = messages # store in edata + funcs[(srctype, etype, dsttype)] = ( + fn.copy_e(etype, "m"), + fn.sum("m", "h"), + ) # define message and reduce functions + + g.multi_update_all( + funcs, "sum" + ) # update all, reduce by first type-wisely then across different types + feature_dict = {} + for ntype in g.ntypes: + h = F.normalize(g.nodes[ntype].data["h"], dim=1, p=2) # l2 normalize + feature_dict[ntype] = h + return feature_dict class Model(nn.Module): - def __init__(self, user_size, item_size, hidden_size, num_layers=3, device=None): + def __init__(self, g, in_size, num_layers, lambda_reg, device=None): super(Model, self).__init__() - self.user_size = user_size - self.item_size = item_size - self.hidden_size = hidden_size - self.embedding_weights = self._init_weights() - self.layers = nn.ModuleList([GCNLayer() for _ in range(num_layers)]) + self.norm_dict = dict() + self.lambda_reg = lambda_reg self.device = device - def forward(self, graph): - user_embedding = self.embedding_weights["user_embedding"] - item_embedding = self.embedding_weights["item_embedding"] + for srctype, etype, dsttype in g.canonical_etypes: + src, dst = g.edges(etype=(srctype, etype, dsttype)) + dst_degree = g.in_degrees( + dst, etype=(srctype, etype, dsttype) + ).float() # obtain degrees + src_degree = g.out_degrees(src, etype=(srctype, etype, dsttype)).float() + norm = torch.pow(src_degree * dst_degree, -0.5).unsqueeze(1) # compute norm + self.norm_dict[(srctype, etype, dsttype)] = norm - for i, layer in enumerate(self.layers, start=1): - if i == 1: - embeddings = layer(graph, user_embedding, item_embedding) - else: - embeddings = layer( - graph, embeddings[: self.user_size], embeddings[self.user_size:] - ) - - user_embedding = user_embedding + embeddings[: self.user_size] * ( - 1 / (i + 1) - ) - item_embedding = item_embedding + embeddings[self.user_size:] * ( - 1 / (i + 1) - ) - - return user_embedding, item_embedding + self.layers = nn.ModuleList([GCNLayer(self.norm_dict) for _ in range(num_layers)]) - def _init_weights(self): - initializer = nn.init.xavier_uniform_ + self.initializer = nn.init.xavier_uniform_ - weights_dict = nn.ParameterDict( + # embeddings for different types of nodes + self.feature_dict = nn.ParameterDict( { - "user_embedding": nn.Parameter( - initializer(torch.empty(self.user_size, self.hidden_size)) - ), - "item_embedding": nn.Parameter( - initializer(torch.empty(self.item_size, self.hidden_size)) - ), + ntype: nn.Parameter( + self.initializer(torch.empty(g.num_nodes(ntype), in_size)) + ) + for ntype in g.ntypes } ) - return weights_dict + + def forward(self, g, users=None, pos_items=None, neg_items=None): + h_dict = {ntype: self.feature_dict[ntype] for ntype in g.ntypes} + # obtain features of each layer and concatenate them all + user_embeds = h_dict[USER_KEY] + item_embeds = h_dict[ITEM_KEY] + + for k, layer in enumerate(self.layers): + h_dict = layer(g, h_dict) + user_embeds = user_embeds + (h_dict[USER_KEY] * 1 / (k + 1)) + item_embeds = item_embeds + (h_dict[ITEM_KEY] * 1 / (k + 1)) + + u_g_embeddings = user_embeds if users is None else user_embeds[users, :] + pos_i_g_embeddings = item_embeds if pos_items is None else item_embeds[pos_items, :] + neg_i_g_embeddings = item_embeds if neg_items is None else item_embeds[neg_items, :] + + return u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings + + def loss_fn(self, users, pos_items, neg_items): + pos_scores = (users * pos_items).sum(1) + neg_scores = (users * neg_items).sum(1) + + bpr_loss = F.softplus(neg_scores - pos_scores).mean() + reg_loss = ( + (1 / 2) + * ( + torch.norm(users) ** 2 + + torch.norm(pos_items) ** 2 + + torch.norm(neg_items) ** 2 + ) + / len(users) + ) + + return bpr_loss + self.lambda_reg * reg_loss, bpr_loss, reg_loss diff --git a/cornac/models/lightgcn/recom_lightgcn.py b/cornac/models/lightgcn/recom_lightgcn.py index a389b295a..effd015f2 100644 --- a/cornac/models/lightgcn/recom_lightgcn.py +++ b/cornac/models/lightgcn/recom_lightgcn.py @@ -19,7 +19,7 @@ from tqdm.auto import tqdm, trange -class LightGCN1(Recommender): +class LightGCN(Recommender): """ LightGCN @@ -28,8 +28,11 @@ class LightGCN1(Recommender): name: string, default: 'LightGCN' The name of the recommender model. + emb_size: int, default: 64 + Size of the node embeddings. + num_epochs: int, default: 1000 - Maximum number of iterations or the number of epochs + Maximum number of iterations or the number of epochs. learning_rate: float, default: 0.001 The learning rate that determines the step size at each iteration @@ -37,15 +40,6 @@ class LightGCN1(Recommender): train_batch_size: int, default: 1024 Mini-batch size used for train set - test_batch_size: int, default: 100 - Mini-batch size used for test set - - hidden_dim: int, default: 64 - The embedding size of the model - - num_layers: int, default: 3 - Number of LightGCN Layers - early_stopping: {min_delta: float, patience: int}, optional, default: None If `None`, no early stopping. Meaning of the arguments: @@ -72,19 +66,16 @@ class LightGCN1(Recommender): References ---------- - * He, X., Deng, K., Wang, X., Li, Y., Zhang, Y., & Wang, M. (2020). - LightGCN: Simplifying and Powering Graph Convolution Network for - Recommendation. + * Wang, Xiang, et al. "Neural graph collaborative filtering." Proceedings of the 42nd international ACM SIGIR conference on Research and development in Information Retrieval. 2019. """ def __init__( self, name="LightGCN", + emb_size=64, num_epochs=1000, learning_rate=0.001, train_batch_size=1024, - test_batch_size=100, - hidden_dim=64, num_layers=3, early_stopping=None, lambda_reg=1e-4, @@ -93,13 +84,11 @@ def __init__( seed=2020, ): super().__init__(name=name, trainable=trainable, verbose=verbose) - + self.emb_size = emb_size self.num_epochs = num_epochs self.learning_rate = learning_rate - self.hidden_dim = hidden_dim - self.num_layers = num_layers self.train_batch_size = train_batch_size - self.test_batch_size = test_batch_size + self.num_layers = num_layers self.early_stopping = early_stopping self.lambda_reg = lambda_reg self.seed = seed @@ -135,19 +124,15 @@ def fit(self, train_set, val_set=None): if torch.cuda.is_available(): torch.cuda.manual_seed_all(self.seed) + graph = construct_graph(train_set).to(self.device) model = Model( - train_set.total_users, - train_set.total_items, - self.hidden_dim, + graph, + self.emb_size, self.num_layers, + self.lambda_reg, ).to(self.device) - graph = construct_graph(train_set).to(self.device) - - optimizer = torch.optim.Adam( - model.parameters(), lr=self.learning_rate, weight_decay=self.lambda_reg - ) - loss_fn = torch.nn.BCELoss(reduction="sum") + optimizer = torch.optim.Adam(model.parameters(), lr=self.learning_rate) # model training pbar = trange( @@ -172,26 +157,17 @@ def fit(self, train_set, val_set=None): position=1, disable=not self.verbose, ): - user_embeddings, item_embeddings = model(graph) - - batch_u = torch.from_numpy(batch_u).long().to(self.device) - batch_i = torch.from_numpy(batch_i).long().to(self.device) - batch_j = torch.from_numpy(batch_j).long().to(self.device) - - user_embed = user_embeddings[batch_u] - positive_item_embed = item_embeddings[batch_i] - negative_item_embed = item_embeddings[batch_j] - - ui_scores = (user_embed * positive_item_embed).sum(dim=1) - uj_scores = (user_embed * negative_item_embed).sum(dim=1) + u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings = model( + graph, batch_u, batch_i, batch_j + ) - loss = loss_fn( - torch.sigmoid(ui_scores - uj_scores), torch.ones_like(ui_scores) + batch_loss, batch_bpr_loss, batch_reg_loss = model.loss_fn( + u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings ) - accum_loss += loss.cpu().item() + accum_loss += batch_loss.cpu().item() * len(batch_u) optimizer.zero_grad() - loss.backward() + batch_loss.backward() optimizer.step() accum_loss /= len(train_set.uir_tuple[0]) # normalize over all observations @@ -199,17 +175,16 @@ def fit(self, train_set, val_set=None): # store user and item embedding matrices for prediction model.eval() - self.U, self.V = model(graph) + u_embs, i_embs, _ = model(graph) + # we will use numpy for faster prediction in the score function, no need torch + self.U = u_embs.cpu().detach().numpy() + self.V = i_embs.cpu().detach().numpy() if self.early_stopping is not None and self.early_stop( **self.early_stopping ): break - # we will use numpy for faster prediction in the score function, no need torch - self.U = self.U.cpu().detach().numpy() - self.V = self.V.cpu().detach().numpy() - def monitor_value(self): """Calculating monitored value used for early stopping on validation set (`val_set`). This function will be called by `early_stop()` function. @@ -223,38 +198,17 @@ def monitor_value(self): if self.val_set is None: return None - import torch + from ...metrics import Recall + from ...eval_methods import ranking_eval - loss_fn = torch.nn.BCELoss(reduction="sum") - accum_loss = 0.0 - pbar = tqdm( - self.val_set.uij_iter(batch_size=self.test_batch_size), - desc="Validation", - total=self.val_set.num_batches(self.test_batch_size), - leave=False, - position=1, - disable=not self.verbose, - ) - for batch_u, batch_i, batch_j in pbar: - batch_u = torch.from_numpy(batch_u).long().to(self.device) - batch_i = torch.from_numpy(batch_i).long().to(self.device) - batch_j = torch.from_numpy(batch_j).long().to(self.device) - - user_embed = self.U[batch_u] - positive_item_embed = self.V[batch_i] - negative_item_embed = self.V[batch_j] - - ui_scores = (user_embed * positive_item_embed).sum(dim=1) - uj_scores = (user_embed * negative_item_embed).sum(dim=1) - - loss = loss_fn( - torch.sigmoid(ui_scores - uj_scores), torch.ones_like(ui_scores) - ) - accum_loss += loss.cpu().item() - pbar.set_postfix(val_loss=accum_loss) - - accum_loss /= len(self.val_set.uir_tuple[0]) - return -accum_loss # higher is better -> smaller loss is better + recall_20 = ranking_eval( + model=self, + metrics=[Recall(k=20)], + train_set=self.train_set, + test_set=self.val_set + )[0][0] + + return recall_20 # Section 4.1.2 in the paper, same strategy as NGCF. def score(self, user_idx, item_idx=None): """Predict the scores/ratings of a user for an item. diff --git a/cornac/models/lightgcn2/__init__.py b/cornac/models/lightgcn2/__init__.py deleted file mode 100644 index 0d239a783..000000000 --- a/cornac/models/lightgcn2/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2018 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -from .recom_lightgcn import LightGCN diff --git a/cornac/models/lightgcn2/lightgcn.py b/cornac/models/lightgcn2/lightgcn.py deleted file mode 100644 index 47172301c..000000000 --- a/cornac/models/lightgcn2/lightgcn.py +++ /dev/null @@ -1,129 +0,0 @@ -# Reference: https://github.com/dmlc/dgl/blob/master/examples/pytorch/NGCF/NGCF/model.py - -import torch -import torch.nn as nn -import torch.nn.functional as F -import dgl -import dgl.function as fn - - -USER_KEY = "user" -ITEM_KEY = "item" - - -def construct_graph(data_set): - """ - Generates graph given a cornac data set - - Parameters - ---------- - data_set : cornac.data.dataset.Dataset - The data set as provided by cornac - """ - user_indices, item_indices, _ = data_set.uir_tuple - - data_dict = { - (USER_KEY, "user_item", ITEM_KEY): (user_indices, item_indices), - (ITEM_KEY, "item_user", USER_KEY): (item_indices, user_indices), - } - num_dict = {USER_KEY: data_set.total_users, ITEM_KEY: data_set.total_items} - - return dgl.heterograph(data_dict, num_nodes_dict=num_dict) - - -class GCNLayer(nn.Module): - def __init__(self, norm_dict): - super(GCNLayer, self).__init__() - - # norm - self.norm_dict = norm_dict - - def forward(self, g, feat_dict): - funcs = {} # message and reduce functions dict - # for each type of edges, compute messages and reduce them all - for srctype, etype, dsttype in g.canonical_etypes: - src, dst = g.edges(etype=(srctype, etype, dsttype)) - norm = self.norm_dict[(srctype, etype, dsttype)] - # TODO: CHECK HERE - messages = norm * feat_dict[srctype][src] # compute messages - g.edges[(srctype, etype, dsttype)].data[ - etype - ] = messages # store in edata - funcs[(srctype, etype, dsttype)] = ( - fn.copy_e(etype, "m"), - fn.sum("m", "h"), - ) # define message and reduce functions - - g.multi_update_all( - funcs, "sum" - ) # update all, reduce by first type-wisely then across different types - feature_dict = {} - for ntype in g.ntypes: - h = F.normalize(g.nodes[ntype].data["h"], dim=1, p=2) # l2 normalize - feature_dict[ntype] = h - return feature_dict - - -class Model(nn.Module): - def __init__(self, g, in_size, num_layers, lambda_reg, device=None): - super(Model, self).__init__() - self.norm_dict = dict() - self.lambda_reg = lambda_reg - self.device = device - - for srctype, etype, dsttype in g.canonical_etypes: - src, dst = g.edges(etype=(srctype, etype, dsttype)) - dst_degree = g.in_degrees( - dst, etype=(srctype, etype, dsttype) - ).float() # obtain degrees - src_degree = g.out_degrees(src, etype=(srctype, etype, dsttype)).float() - norm = torch.pow(src_degree * dst_degree, -0.5).unsqueeze(1) # compute norm - self.norm_dict[(srctype, etype, dsttype)] = norm - - self.layers = nn.ModuleList([GCNLayer(self.norm_dict) for _ in range(num_layers)]) - - self.initializer = nn.init.xavier_uniform_ - - # embeddings for different types of nodes - self.feature_dict = nn.ParameterDict( - { - ntype: nn.Parameter( - self.initializer(torch.empty(g.num_nodes(ntype), in_size)) - ) - for ntype in g.ntypes - } - ) - - def forward(self, g, users=None, pos_items=None, neg_items=None): - h_dict = {ntype: self.feature_dict[ntype] for ntype in g.ntypes} - # obtain features of each layer and concatenate them all - user_embeds = h_dict[USER_KEY] - item_embeds = h_dict[ITEM_KEY] - - for k, layer in enumerate(self.layers): - h_dict = layer(g, h_dict) - user_embeds = user_embeds + (h_dict[USER_KEY] * 1 / (k + 1)) - item_embeds = item_embeds + (h_dict[ITEM_KEY] * 1 / (k + 1)) - - u_g_embeddings = user_embeds if users is None else user_embeds[users, :] - pos_i_g_embeddings = item_embeds if pos_items is None else item_embeds[pos_items, :] - neg_i_g_embeddings = item_embeds if neg_items is None else item_embeds[neg_items, :] - - return u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings - - def loss_fn(self, users, pos_items, neg_items): - pos_scores = (users * pos_items).sum(1) - neg_scores = (users * neg_items).sum(1) - - bpr_loss = F.softplus(neg_scores - pos_scores).mean() - reg_loss = ( - (1 / 2) - * ( - torch.norm(users) ** 2 - + torch.norm(pos_items) ** 2 - + torch.norm(neg_items) ** 2 - ) - / len(users) - ) - - return bpr_loss + self.lambda_reg * reg_loss, bpr_loss, reg_loss diff --git a/cornac/models/lightgcn2/recom_lightgcn.py b/cornac/models/lightgcn2/recom_lightgcn.py deleted file mode 100644 index effd015f2..000000000 --- a/cornac/models/lightgcn2/recom_lightgcn.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright 2018 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -from ..recommender import Recommender -from ...exception import ScoreException - -from tqdm.auto import tqdm, trange - - -class LightGCN(Recommender): - """ - LightGCN - - Parameters - ---------- - name: string, default: 'LightGCN' - The name of the recommender model. - - emb_size: int, default: 64 - Size of the node embeddings. - - num_epochs: int, default: 1000 - Maximum number of iterations or the number of epochs. - - learning_rate: float, default: 0.001 - The learning rate that determines the step size at each iteration - - train_batch_size: int, default: 1024 - Mini-batch size used for train set - - early_stopping: {min_delta: float, patience: int}, optional, default: None - If `None`, no early stopping. Meaning of the arguments: - - - `min_delta`: the minimum increase in monitored value on validation - set to be considered as improvement, - i.e. an increment of less than min_delta will count as - no improvement. - - - `patience`: number of epochs with no improvement after which - training should be stopped. - - lambda_reg: float, default: 1e-4 - Weight decay for the L2 normalization - - trainable: boolean, optional, default: True - When False, the model is not trained and Cornac assumes that the model - is already pre-trained. - - verbose: boolean, optional, default: False - When True, some running logs are displayed. - - seed: int, optional, default: 2020 - Random seed for parameters initialization. - - References - ---------- - * Wang, Xiang, et al. "Neural graph collaborative filtering." Proceedings of the 42nd international ACM SIGIR conference on Research and development in Information Retrieval. 2019. - """ - - def __init__( - self, - name="LightGCN", - emb_size=64, - num_epochs=1000, - learning_rate=0.001, - train_batch_size=1024, - num_layers=3, - early_stopping=None, - lambda_reg=1e-4, - trainable=True, - verbose=False, - seed=2020, - ): - super().__init__(name=name, trainable=trainable, verbose=verbose) - self.emb_size = emb_size - self.num_epochs = num_epochs - self.learning_rate = learning_rate - self.train_batch_size = train_batch_size - self.num_layers = num_layers - self.early_stopping = early_stopping - self.lambda_reg = lambda_reg - self.seed = seed - - def fit(self, train_set, val_set=None): - """Fit the model to observations. - - Parameters - ---------- - train_set: :obj:`cornac.data.Dataset`, required - User-Item preference data as well as additional modalities. - - val_set: :obj:`cornac.data.Dataset`, optional, default: None - User-Item preference data for model selection purposes (e.g., early stopping). - - Returns - ------- - self : object - """ - Recommender.fit(self, train_set, val_set) - - if not self.trainable: - return self - - # model setup - import torch - from .lightgcn import Model - from .lightgcn import construct_graph - - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - if self.seed is not None: - torch.manual_seed(self.seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(self.seed) - - graph = construct_graph(train_set).to(self.device) - model = Model( - graph, - self.emb_size, - self.num_layers, - self.lambda_reg, - ).to(self.device) - - optimizer = torch.optim.Adam(model.parameters(), lr=self.learning_rate) - - # model training - pbar = trange( - self.num_epochs, - desc="Training", - unit="iter", - position=0, - leave=False, - disable=not self.verbose, - ) - for _ in pbar: - model.train() - accum_loss = 0.0 - for batch_u, batch_i, batch_j in tqdm( - train_set.uij_iter( - batch_size=self.train_batch_size, - shuffle=True, - ), - desc="Epoch", - total=train_set.num_batches(self.train_batch_size), - leave=False, - position=1, - disable=not self.verbose, - ): - u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings = model( - graph, batch_u, batch_i, batch_j - ) - - batch_loss, batch_bpr_loss, batch_reg_loss = model.loss_fn( - u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings - ) - accum_loss += batch_loss.cpu().item() * len(batch_u) - - optimizer.zero_grad() - batch_loss.backward() - optimizer.step() - - accum_loss /= len(train_set.uir_tuple[0]) # normalize over all observations - pbar.set_postfix(loss=accum_loss) - - # store user and item embedding matrices for prediction - model.eval() - u_embs, i_embs, _ = model(graph) - # we will use numpy for faster prediction in the score function, no need torch - self.U = u_embs.cpu().detach().numpy() - self.V = i_embs.cpu().detach().numpy() - - if self.early_stopping is not None and self.early_stop( - **self.early_stopping - ): - break - - def monitor_value(self): - """Calculating monitored value used for early stopping on validation set (`val_set`). - This function will be called by `early_stop()` function. - - Returns - ------- - res : float - Monitored value on validation set. - Return `None` if `val_set` is `None`. - """ - if self.val_set is None: - return None - - from ...metrics import Recall - from ...eval_methods import ranking_eval - - recall_20 = ranking_eval( - model=self, - metrics=[Recall(k=20)], - train_set=self.train_set, - test_set=self.val_set - )[0][0] - - return recall_20 # Section 4.1.2 in the paper, same strategy as NGCF. - - def score(self, user_idx, item_idx=None): - """Predict the scores/ratings of a user for an item. - - Parameters - ---------- - user_idx: int, required - The index of the user for whom to perform score prediction. - - item_idx: int, optional, default: None - The index of the item for which to perform score prediction. - If None, scores for all known items will be returned. - - Returns - ------- - res : A scalar or a Numpy array - Relative scores that the user gives to the item or to all known items - - """ - if item_idx is None: - if self.train_set.is_unk_user(user_idx): - raise ScoreException( - "Can't make score prediction for (user_id=%d)" % user_idx - ) - known_item_scores = self.V.dot(self.U[user_idx, :]) - return known_item_scores - else: - if self.train_set.is_unk_user(user_idx) or self.train_set.is_unk_item( - item_idx - ): - raise ScoreException( - "Can't make score prediction for (user_id=%d, item_id=%d)" - % (user_idx, item_idx) - ) - return self.V[item_idx, :].dot(self.U[user_idx, :]) diff --git a/cornac/models/lightgcn2/requirements.txt b/cornac/models/lightgcn2/requirements.txt deleted file mode 100644 index 32f294fbc..000000000 --- a/cornac/models/lightgcn2/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torch>=2.0.0 -dgl>=1.1.0 \ No newline at end of file From 1d6c90ee0650997cdad40880b4efc7149737d53a Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Tue, 17 Oct 2023 11:54:32 +0800 Subject: [PATCH 21/23] update docstring --- cornac/models/lightgcn/recom_lightgcn.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cornac/models/lightgcn/recom_lightgcn.py b/cornac/models/lightgcn/recom_lightgcn.py index effd015f2..5658da656 100644 --- a/cornac/models/lightgcn/recom_lightgcn.py +++ b/cornac/models/lightgcn/recom_lightgcn.py @@ -30,7 +30,7 @@ class LightGCN(Recommender): emb_size: int, default: 64 Size of the node embeddings. - + num_epochs: int, default: 1000 Maximum number of iterations or the number of epochs. @@ -40,6 +40,9 @@ class LightGCN(Recommender): train_batch_size: int, default: 1024 Mini-batch size used for train set + num_layers: int, default: 3 + Number of LightGCN Layers + early_stopping: {min_delta: float, patience: int}, optional, default: None If `None`, no early stopping. Meaning of the arguments: From 0c81202c77c76c2631d723b29f296928a8aea9ad Mon Sep 17 00:00:00 2001 From: Darryl Ong Date: Tue, 17 Oct 2023 12:00:13 +0800 Subject: [PATCH 22/23] change param name to 'batch_size' --- cornac/models/lightgcn/recom_lightgcn.py | 10 +++++----- examples/lightgcn_example.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cornac/models/lightgcn/recom_lightgcn.py b/cornac/models/lightgcn/recom_lightgcn.py index 5658da656..3e4f98ad4 100644 --- a/cornac/models/lightgcn/recom_lightgcn.py +++ b/cornac/models/lightgcn/recom_lightgcn.py @@ -37,7 +37,7 @@ class LightGCN(Recommender): learning_rate: float, default: 0.001 The learning rate that determines the step size at each iteration - train_batch_size: int, default: 1024 + batch_size: int, default: 1024 Mini-batch size used for train set num_layers: int, default: 3 @@ -78,7 +78,7 @@ def __init__( emb_size=64, num_epochs=1000, learning_rate=0.001, - train_batch_size=1024, + batch_size=1024, num_layers=3, early_stopping=None, lambda_reg=1e-4, @@ -90,7 +90,7 @@ def __init__( self.emb_size = emb_size self.num_epochs = num_epochs self.learning_rate = learning_rate - self.train_batch_size = train_batch_size + self.batch_size = batch_size self.num_layers = num_layers self.early_stopping = early_stopping self.lambda_reg = lambda_reg @@ -151,11 +151,11 @@ def fit(self, train_set, val_set=None): accum_loss = 0.0 for batch_u, batch_i, batch_j in tqdm( train_set.uij_iter( - batch_size=self.train_batch_size, + batch_size=self.batch_size, shuffle=True, ), desc="Epoch", - total=train_set.num_batches(self.train_batch_size), + total=train_set.num_batches(self.batch_size), leave=False, position=1, disable=not self.verbose, diff --git a/examples/lightgcn_example.py b/examples/lightgcn_example.py index b91660063..48789ebea 100644 --- a/examples/lightgcn_example.py +++ b/examples/lightgcn_example.py @@ -39,7 +39,7 @@ num_epochs=1000, num_layers=3, early_stopping={"min_delta": 1e-4, "patience": 50}, - train_batch_size=1024, + batch_size=1024, learning_rate=0.001, lambda_reg=1e-4, verbose=True From 3992661e504be212ba28ff9b3a0eda613f41027b Mon Sep 17 00:00:00 2001 From: Quoc-Tuan Truong Date: Tue, 17 Oct 2023 04:26:00 +0000 Subject: [PATCH 23/23] Fix paper reference --- cornac/models/lightgcn/lightgcn.py | 2 -- cornac/models/lightgcn/recom_lightgcn.py | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cornac/models/lightgcn/lightgcn.py b/cornac/models/lightgcn/lightgcn.py index 47172301c..eedd72b42 100644 --- a/cornac/models/lightgcn/lightgcn.py +++ b/cornac/models/lightgcn/lightgcn.py @@ -1,5 +1,3 @@ -# Reference: https://github.com/dmlc/dgl/blob/master/examples/pytorch/NGCF/NGCF/model.py - import torch import torch.nn as nn import torch.nn.functional as F diff --git a/cornac/models/lightgcn/recom_lightgcn.py b/cornac/models/lightgcn/recom_lightgcn.py index 3e4f98ad4..635fb67a3 100644 --- a/cornac/models/lightgcn/recom_lightgcn.py +++ b/cornac/models/lightgcn/recom_lightgcn.py @@ -69,7 +69,9 @@ class LightGCN(Recommender): References ---------- - * Wang, Xiang, et al. "Neural graph collaborative filtering." Proceedings of the 42nd international ACM SIGIR conference on Research and development in Information Retrieval. 2019. + * He, X., Deng, K., Wang, X., Li, Y., Zhang, Y., & Wang, M. (2020). + LightGCN: Simplifying and Powering Graph Convolution Network for + Recommendation. """ def __init__(