-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathutils.py
183 lines (156 loc) · 7.28 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
import os
import random
import pickle
import functools
import warnings
import multiprocessing
from collections import deque, defaultdict
import tqdm
import numpy as np
import pandas as pd
import scipy.sparse as sp
import torch
def add_argument(parser):
parser.add_argument('--root', type=str, default='data')
parser.add_argument('--model', type=str, default='experimental',
choices=['just', 'experimental', 'hin2vec', 'heer', 'metapath2vec', 'deepwalk', 'LINE'])
parser.add_argument('--dataset', type=str, default='dblp',
choices=['blog-catalog', 'douban_movie', 'dblp', 'yago']+['synthetic_%d_%d' % (n, t) for n in [100, 1000, 10000] for t in [2, 4, 8]])
parser.add_argument('--epoch', type=int, default=10)
parser.add_argument('--batch_size', type=int, default=128)
parser.add_argument('--d', type=int, default=128)
parser.add_argument('--l', type=int, default=100)
parser.add_argument('--k', type=int, default=5)
parser.add_argument('--m', type=int, default=5)
parser.add_argument('--alpha', type=float, default=0.05)
parser.add_argument('--que_size', type=int, default=2)
parser.add_argument('--lr', type=float, default=0.025)
parser.add_argument('--lr2', type=float, default=0.0025)
parser.add_argument('--restore', action='store_true')
parser.add_argument('--approximate_naive', action='store_true')
def get_name(args):
if args.model == 'experimental':
return 'experimental_%s_%d_%d_%d_%d_%.2f_%.6f_%6f' % (args.dataset, args.d, args.l, args.k, args.m, args.alpha, args.lr, args.lr2)
elif args.model == 'just':
name = 'just_%s_%d_%d_%d_%d' % (args.dataset, args.d, args.l, args.k, args.m)
name += '_%.2f_%d' % (args.alpha, args.que_size)
return name
else:
return '%s_%s_%d_%.4f' % (args.model, args.dataset, args.d, args.lr)
def deprecated(replacement=None):
def outer(fun):
msg = '%s is deprecated' % fun.__name__
if replacement is not None:
msg += '; use %s instead' % replacement
if fun.__doc__ is None:
fun.__doc__ = msg
@functools.wraps(fun)
def inner(*args, **kwargs):
warnings.warn(msg, category=DeprecationWarning, stacklevel=1)
return fun(*args, **kwargs)
return inner
return outer
def convert_defaultdict_to_dict(x):
"""nested defaultdict을 dict으로 변환."""
# Not working if list exist
if isinstance(x, defaultdict):
x = {k: convert_defaultdict_to_dict(v) for k, v in x.items()}
return x
def create_graph(edge_df, node_num, type_order):
# Indice information
graph = defaultdict(lambda : defaultdict(set))
for idx, v in edge_df.groupby(by=['v1', 't2'])['v2'].apply(set).iteritems():
graph[idx[0]][idx[1]] = graph[idx[0]][idx[1]].union(v)
for idx, v in edge_df.groupby(by=['v2', 't1'])['v1'].apply(set).iteritems():
graph[idx[0]][idx[1]] = graph[idx[0]][idx[1]].union(v)
# Compact representation of graph
adj_data = [graph[x] for x in range(node_num)]
adj_data = [[x[y] for y in type_order] for x in adj_data]
adj_size = [[len(y) for y in x] for x in adj_data]
adj_start = [[None]*len(type_order) for _ in range(node_num)]
count = 0
for i in range(node_num):
for j in range(len(type_order)):
adj_start[i][j] = count
count += adj_size[i][j]
adj_data = [item for sublist in adj_data for subsublist in sublist for item in subsublist]
return adj_data, adj_size, adj_start
def make_dot(var, params):
""" Produces Graphviz representation of PyTorch autograd graph
Blue nodes are the Variables that require grad, orange are Tensors
saved for backward in torch.autograd.Function
Args:
var: output Variable
params: dict of (name, Variable) to add names to node that
require grad (TODO: make optional)
"""
param_map = {id(v): k for k, v in params.items()}
node_attr = dict(style='filled',
shape='box',
align='left',
fontsize='12',
ranksep='0.1',
height='0.2')
dot = Digraph(node_attr=node_attr, graph_attr=dict(size="12,12"))
seen = set()
def size_to_str(size):
return '('+(', ').join(['%d'% v for v in size])+')'
def add_nodes(var):
if var not in seen:
if torch.is_tensor(var):
dot.node(str(id(var)), size_to_str(var.size()), fillcolor='orange')
elif hasattr(var, 'variable'):
u = var.variable
node_name = '%s\n %s' % (param_map.get(id(u)), size_to_str(u.size()))
dot.node(str(id(var)), node_name, fillcolor='lightblue')
else:
dot.node(str(id(var)), str(type(var).__name__))
seen.add(var)
if hasattr(var, 'next_functions'):
for u in var.next_functions:
if u[0] is not None:
dot.edge(str(id(u[0])), str(id(var)))
add_nodes(u[0])
if hasattr(var, 'saved_tensors'):
for t in var.saved_tensors:
dot.edge(str(id(t)), str(id(var)))
add_nodes(t)
add_nodes(var.grad_fn)
return dot
def load_metadata(data_dir):
with open(os.path.join(data_dir, 'node_type.pickle'), 'rb') as f:
node_type = pickle.load(f)
return node_type
def load_data(args):
data_dir = os.path.join(args.root, args.dataset)
node_df, test_edge_df = None, None
if args.dataset == 'blog-catalog':
node_type = load_metadata(data_dir)
edge_df = pd.read_csv(os.path.join(data_dir, 'edge.csv'), sep='\t')
test_edge_df = pd.read_csv(os.path.join(data_dir, 'test_edge.csv'), sep='\t')
elif args.dataset == 'douban_movie':
node_type = load_metadata(data_dir)
edge_df = pd.read_csv(os.path.join(data_dir, 'edge.csv'), sep='\t')
test_edge_df = pd.read_csv(os.path.join(data_dir, 'test_edge.csv'), sep='\t')
elif args.dataset == 'dblp':
node_type = load_metadata(data_dir)
node_df = pd.read_csv(os.path.join(data_dir, 'node.csv'), sep='\t')
edge_df = pd.read_csv(os.path.join(data_dir, 'edge.csv'), sep='\t')
elif args.dataset == 'yago':
with open(os.path.join(data_dir, 'node_type.pickle'), 'rb') as f:
node_type = pickle.load(f)
edge_df = pd.read_csv(os.path.join(data_dir, 'train_edge.csv'), sep='\t')
test_edge_df = pd.read_csv(os.path.join(data_dir, 'test_edge.csv'), sep='\t')
elif args.dataset.startswith('synthetic'):
node_num, type_num = args.dataset.split('_')[1:]
with open(os.path.join(args.root, 'synthetic', 'node_type_'+node_num+'_'+type_num+'.pickle'), 'rb') as f:
node_type = pickle.load(f)
edge_df = pd.read_csv(os.path.join(args.root, 'synthetic', 'edge_'+node_num+'_'+type_num+'.csv'), sep='\t')
else:
raise Exception("Undefined dataset")
return node_type, edge_df, node_df, test_edge_df
def create_adjacency_matrix(adj_list: dict, shape: (int, int)):
row = [k for k, v in adj_list.items() for _ in v]
col = [x for _, v in adj_list.items() for x in v]
adj_matrix = sp.coo_matrix((np.ones(len(row)), (row, col)), shape=shape).tocsr()
return adj_matrix