Training huge hdf5 file in batches trigger DeferredInitializationError

[mickymount]

Hi All,

I am a newbie to mxnet, however, I did follow couple of tutorials online and thought of training a model on a huge hdf5 file ndarrays not images. But, I got hanged up on "DeferredInitilizatioError". However, from the tutorials and the google searches I looked at tells me this representation is correct... Is there something I am missing in the code? Thank you in advance...

import numpy as np
from memory_profiler import profile
from d2l import mxnet as d2l
import mxnet as mx
from mxnet import autograd, gluon, init, npx
from mxnet.gluon import nn
import h5py
import os
import matplotlib.pyplot as plt


npx.set_np()
os.getcwd()

class HDF5Dataset(mx.gluon.data.Dataset):
	def __init__(self, filepath, datasets):
		"""
		A Gluon Dataset for data in a HDF5 file. Can use multiple datasets of equal length.
		param filepath: path to .hdf5 file
		param datasets: list of strings denoting datasets to use. Order used for __getitem__ return.
		"""
		self._file = f = h5py.File(filepath, 'r')
		self._datasets = [self._file[dataset] for dataset in datasets]
		assert len(set([len(dataset) for dataset in self._datasets])) == 1, "Check datasets are equal length."
		
	def __getitem__(self, idx):
		return tuple(dataset[idx] for dataset in self._datasets)
	
	def __len__(self):
		return len(self._datasets[0])
		
	def close(self):
		self._file.close()


#@save
def readData():
	dataset = HDF5Dataset('train1.h5', datasets=['features', 'target'])

	dataloader = mx.gluon.data.DataLoader(dataset, batch_size=16384, shuffle=True, num_workers=0)
	return dataloader
def sgd(params, states, hyperparams):
	for p in params:
		p[:] -= hyperparams['lr'] * p.grad


def creatNet():
	net = gluon.nn.Sequential()
	with net.name_scope():
		net.add(gluon.nn.Dense(1))
	return net

#@save
def train_concise_ch11(tr_name, hyperparams, data_iter, num_epochs=2):
	# Initialization
	ctx =  mx.gpu() if mx.context.num_gpus() else mx.cpu()
	net = creatNet()

	print("intialize sequential()")
	net.initialize(init=init.Normal(sigma=0.01), force_reinit=True, ctx=ctx)

	print(net[0])
	print("add sigma")

	net.cast("float64")

	print("cast net float64")
	loss = gluon.loss.L2Loss()
	trainer = gluon.Trainer(net.collect_params(), tr_name, hyperparams)
	#net.collect_params('.*bias')
	print("trainer produced")
	

	print("call loss function")
	#animator = d2l.Animator(xlabel='epoch', ylabel='loss', xlim=[0, num_epochs], ylim=[0.22, 0.35])
	n, timer = 0, d2l.Timer()
	print("set timer")
	for _ in range(num_epochs):
		cumulative_train_loss = mx.nd.zeros(1, ctx=ctx)
		training_samples = 0
		print("first epochs")
		for batch_indx, (X, y) in enumerate(data_iter):
			print("X.shape = {}, y.shape = {}".format(X.shape, y.shape))
			X = X.as_in_context(mx.cpu())
			y = y.as_in_context(mx.cpu())

			with autograd.record():
				output = net(X)
				l = loss(output, y)
				print("calculated loss")
			l.backward()
			print("goback and call loss ")
			trainer.step(X.shape[0])
			#cumulative_train_loss += loss.sum()
			#training_samples += X.shape[0]
		npx.waitall()
		#train_loss = cumulative_train_loss.asscalar()/training_samples
	print(f'''loss: {animator.Y[0][-1]:.3f},''' '{timer.avg():.3f} sec/epoch')

def main():
 
	print("Is this working")
	
	data_set = readData()
	mx.nd.waitall()
	mx.profiler.set_config(profile_all=True, aggregate_stats=True, filename='profile_output_16gb_bs16k_num_workers0.json')
	mx.profiler.set_state('run')

	train_concise_ch11('sgd', {'learning_rate': 0.05}, data_set)
	
	mx.nd.waitall()
	mx.profiler.set_state('stop')

if __name__ == '__main__' :

	main()

	print("Done training!!!!")

[leezu]

If you don't specify in_units= in gluon.nn.Dense(1), the number of in_units will be inferred from the first forward pass. If you don't want to infer them, just specify the in_units= argument when adding the Dense layer.

[mickymount]

@leezu thanks so much for the quick response, that was trick!!!!