datagen.py

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from collections import Counter
from collections import deque

from numpy.random import PCG64
from scipy.stats import multivariate_t, multivariate_normal


main_generator = np.random.Generator(PCG64(0))
np_data_generator = np.random.Generator(PCG64(0))
# set seed for skeleton generation
def set_seed(i):
	global main_generator
	main_generator = np.random.Generator(PCG64(i))
# set seed for data generation
def set_data_seed(i):
	global np_data_generator
	np_data_generator = np.random.Generator(PCG64(i))



# obtain n uniformly sampled points within a d-sphere with a fixed radius around a given point. Assigns all points to
# given cluster code partially based on code provided here:
# http://extremelearning.com.au/how-to-generate-uniformly-random-points-on-n-spheres-and-n-balls/
def random_ball_num(center, radius, d, n, clunum):
	#print("uniform")
	d = int(d)
	n = int(n)
	u = np_data_generator.normal(0, 1, (n, d + 1))  # an array of d normally distributed random variables
	norm = np.sqrt(np.sum(u[:,:-1] ** 2, 1))
	r = np_data_generator.random(n) ** (1.0 / d)
	normed = np.divide(u, norm[:, None])
	x = r[:, None] * normed
	x[:, :-1] = center + x[:, :-1] * radius
	x[:, -1] = clunum
	return x

def random_ball_num_bias(center, radius, d, n, clunum):
	d = int(d)
	n = int(n)
	u = np_data_generator.normal(0, 1, (n, d + 1))  # an array of d normally distributed random variables
	norm = np.sqrt(np.sum(u ** 2, 1))
	r = np_data_generator.random(n) ** (1.0 / d)
	normed = np.divide(u, norm[:, None])
	x = r[:, None] * normed
	x[:, :-1] = center + x[:, :-1] * radius
	x[:, -1] = clunum
	return x

def gaussian_ball_num(center, radius, d, n, clunum, seed):
	#print("gaussian")
	d = int(d)
	n = int(n)
	center = np.append(center,0)
	cov = np.diag([radius]*(d+1))
	r = multivariate_normal.rvs(size=n, mean=center, cov=cov, random_state=seed)
	x = r.reshape(n,d+1)
	x[:, -1] = clunum
	return x

def t_ball_num(center, radius, d, n, clunum, df, seed):
	#print("student_t")
	d = int(d)
	n = int(n)
	center = np.append(center,0)
	r = multivariate_t.rvs(df=df, size=n, loc=center, shape=radius, random_state=seed)
	x = r.reshape(n,d+1)
	x[:, -1] = clunum
	return x

class densityDataGen:

	def __init__(self, dim=2, clunum=2, clu_ratios=None, core_num=10, min_ratio=0, ratio_noise=0, domain_size=20,
				 radius=1, step=1, ratio_con=0, connections=0, seed=0, dens_factors=False, momentum=0.5,
				 con_momentum=0.9, min_dist=1.1, con_min_dist=0.9, step_spread=0, max_retry=5, verbose=False,
				 safety=True, con_dens_factors=False, con_radius=2, con_step=2, branch=0.05, star=0, square=False,
				 random_start=False, distribution=None, pos=None, con_distribution=None):
		"""
		For extensive parameter explanations, please consult the GitHub README
		:param dim: dimensionality
		:param clunum: cluster number
		:param clu_ratios: cluster ratios
		:param core_num: overall core number
		:param min_ratio: minimal cluster ratio
		:param ratio_noise: noise ratio
		:param domain_size: domain size
		:param radius: core base radius
		:param step: base random walk step size
		:param ratio_con: connection ratio
		:param connections: connections
		:param seed: seed
		:param dens_factors: cluster density factors
		:param momentum: momentum
		:param con_momentum: connection momentum
		:param min_dist: minimal distance ratio between clusters
		:param con_min_dist: minimal distance ratio between connections
		:param step_spread: spread of randomization of random walk step size
		:param max_retry: maximal number of retrys
		:param verbose: verbose mode
		:param safety: safety mode
		:param con_dens_factors: connection density factors
		:param con_radius: connection radius
		:param con_step: connection random walk step size
		:param branch: branching factor
		:param star: star chance
		:param square: square noise distribution
		:param random_start: random start mode
		:param distribution: distribution used for points per core, None default to uniform distribution
		:param pos: specified starting core position for clusters, if
		"""
		self.seed = seed
		set_seed(self.seed)
		self.verbose = verbose
		self.dim = dim
		self.clunum = clunum
		self.clu_ratios = clu_ratios
		self.core_num = core_num
		self.min_ratio = min_ratio
		self.ratio_noise = ratio_noise
		self.domain_size = domain_size
		self.r_sphere = radius
		self.r_step = step
		self.ratio_con = ratio_con
		self.connections = connections
		self.c_sphere = con_radius
		self.c_step = con_step

		self.con_dens_factors = con_dens_factors
		self.dens_factors = dens_factors
		self.step_spread = step_spread
		self.max_retry = max_retry
		self.momentum = momentum
		self.con_momentum = con_momentum
		self.min_dist = min_dist
		self.con_min_dist = con_min_dist
		self.safety = safety

		self.cores = {}
		self.value_space = []
		self.mins = [domain_size + 1] * self.dim
		self.maxs = [-1] * self.dim

		self.stream_content = []
		self.stream_repeat = []
		self.stream_data_cores_block = {}
		self.stream_num_block = {}
		self.noise_block = {}
		self.cur_stream_point = 0
		self.cur_stream_pos = 0
		self.cur_stream_db = {}

		self.cur_block_num = 0
		self.in_reapeat = False

		self.branch = branch
		self.star = star

		self.square = square
		self.random_start = random_start

		self.data_ratio = 1 - ratio_con - ratio_noise

		if (self.data_ratio <= 0):
			raise BaseException("No data points can be generated")

		if pos is None:
			self.pos = [None]*self.clunum
		elif len(pos) == self.clunum:
			for x in pos:
				if x is not None and len(x) != self.dim:
					raise BaseException("Specified position does not match dimensionality")
			self.pos = pos
		else:
			raise BaseException("Shape of cluster positions does not match cluster number")

		self.distributions = []
		if not isinstance(distribution, list):
			distribution = [distribution]*self.clunum

		if len(distribution) != self.clunum:
			raise BaseException("Shape of cluster distributions does not match cluster number")

		for x in distribution:
			if x == None or str(x).lower() == "uniform":
				self.distributions.append("uniform")
			elif str(x).lower() == "paper":
				self.distributions.append("paper")
			elif str(x).lower() == "studentt" or str(x).lower() == "tdist":
				self.distributions.append(2)
			elif str(x).lower() == "gaussian":
				self.distributions.append("gaussian")
			else:
				try:
					self.distributions.append(float(x))
				except:
					raise BaseException(f"{x} is not implemented")

		if con_distribution == None or str(con_distribution).lower() == "uniform":
			self.con_distribution = "uniform"
		elif str(con_distribution).lower() == "paper":
			self.con_distribution = "paper"
		elif str(con_distribution).lower() == "studentt" or str(con_distribution).lower() == "tdist":
			self.con_distribution = 2
		elif str(con_distribution).lower() == "gaussian":
			self.con_distribution = "gaussian"
		else:
			try:
				self.con_distribution = float(con_distribution)
			except:
				raise BaseException(f"{con_distribution} is not implemented")

		if self.clu_ratios == None:
			while (True):
				self.clu_ratios = np.sort(main_generator.random(self.clunum - 1) * self.data_ratio)
				self.clu_ratios = np.append(self.clu_ratios, 1)
				dist = 0
				newrun = False
				for i in range(self.clunum):
					if i==0 and self.clunum == 1:
						dist = 1
						self.clu_ratios[i] = self.data_ratio
					elif i == 0:
						dist = self.clu_ratios[i]
					elif i == self.clunum - 1:
						dist = self.data_ratio - self.clu_ratios[i - 1]
						self.clu_ratios[i] = self.data_ratio
					else:
						dist = self.clu_ratios[i] - self.clu_ratios[i - 1]
					if dist < self.min_ratio:
						newrun = True
						break
				if not newrun:
					break
		elif (self.clunum == len(self.clu_ratios)):
			prior_ratio = 0
			new_ratio = []
			for ratio in self.clu_ratios:
				ratio = ratio + prior_ratio
				new_ratio.append(ratio)
				prior_ratio = ratio

			self.clu_ratios = np.divide(new_ratio, ratio / self.data_ratio)
		else:
			raise BaseException("Shape of cluster ratio does not match cluster number")

		if self.dens_factors == False:
			self.dens_factors = []
			for _ in self.clu_ratios:
				self.dens_factors.append(1)
		elif self.dens_factors == True:
			self.dens_factors = []
			for _ in self.clu_ratios:
				factor = (main_generator.random() * 1.5) + 0.5
				factor = round(factor, 2)
				# print(factor)
				self.dens_factors.append(factor)
		elif (self.clunum != len(self.dens_factors)):
			raise BaseException("Shape of cluster scale factors does not match cluster number")

		if type(self.branch) is list:
			if (self.clunum != len(self.branch)):
				raise BaseException("Shape of branch chances does not match cluster number")
		elif self.branch == "Rand":
			self.branch = []
			for _ in self.clu_ratios:
				branch = main_generator.random() * 0.5
				self.branch.append(branch)
		else:
			branch = self.branch
			self.branch = [branch] * self.clunum

		if type(self.star) is list:
			if (self.clunum != len(self.star)):
				raise BaseException("Shape of star chances does not match cluster number")
		elif self.star == "Rand":
			# print("I was here")
			self.star = []
			for _ in self.clu_ratios:
				star = main_generator.random() * 0.5
				self.star.append(star)
		else:
			star = self.star
			self.star = [star] * self.clunum

		if (self.verbose):
			print("Cluster size ratios:")
			print(self.clu_ratios)
			print("Cluster scale factors:")
			print(self.dens_factors)

		if type(self.momentum) is list:
			if (self.clunum != len(self.momentum)):
				raise BaseException("Shape of cluster momentum does not match cluster number")
			else:
				if (self.verbose):
					print("Cluster momentum factors:")
					print(self.momentum)
		elif self.momentum is None:
			self.momentum = []
			for _ in self.clu_ratios:
				momentum = main_generator.random()
				self.momentum.append(momentum)
			if (self.verbose):
				print("Cluster momentum factors:")
				print(self.momentum)
		else:
			momentum = self.momentum
			self.momentum = [momentum] * self.clunum

		if type(self.core_num) is list:
			if (self.clunum != len(self.core_num)):
				raise BaseException("Shape of cluster core number does not match cluster number")
			else:
				if (self.verbose):
					print("Cluster core numbers:")
					print(self.core_num)

		connection_starts = []
		connection_stops = []

		if type(self.connections) is list:
			for con in self.connections:
				# print(con)
				pair = con.split(";")
				# print(pair)
				# print(pair[0])
				connection_starts.append(int(pair[0]))
				connection_stops.append(int(pair[1]))
		else:
			for i in range(self.connections):
				startclu = main_generator.choice(len(range(self.clunum)), 1)
				stopclu = main_generator.choice(len(range(self.clunum)), 1)
				while (stopclu == startclu):
					stopclu = main_generator.choice(len(range(self.clunum)), 1)
				connection_starts.extend(startclu)
				connection_stops.extend(stopclu)

		if self.con_dens_factors == False:
			self.con_dens_factors = []
			for _ in range(len(connection_starts)):
				self.con_dens_factors.append(1)
		elif self.con_dens_factors == None:
			self.con_dens_factors = []
			for _ in range(len(connection_starts)):
				factor = (main_generator.random() * 1.5) + 0.5
				factor = round(factor, 2)
				self.con_dens_factors.append(factor)
		elif (len(connection_starts) != len(self.con_dens_factors)):
			raise BaseException("Shape of connection scale factors does not match connection number")

		if type(self.con_momentum) is list:
			if (len(connection_starts) != len(self.con_momentum)):
				raise BaseException("Shape of connection momentum does not match connection number")
		elif self.con_momentum == None:
			self.con_momentum = []
			for _ in range(len(connection_starts)):
				con_momentum = main_generator.random()
				self.con_momentum.append(con_momentum)
		else:
			con_momentum = self.con_momentum
			self.con_momentum = [con_momentum] * len(connection_starts)

		if verbose:
			for i in range(len(connection_starts)):
				print("Connection from " + str(connection_starts[i]) + " to " + str(
					connection_stops[i]) + " with factor " + str(self.con_dens_factors[i]) + " and momentum " + str(
					self.con_momentum[i]))

		for cluid in range(self.clunum):
			self.generate_cluster(cluid)

		self.connections = {}
		for i in range(len(connection_starts)):
			retry = 0
			conid = -i - 2
			# print("connection " + str(conid))
			while (True):
				noretry = self.make_connection(connection_starts[i], connection_stops[i], conid)
				if noretry:
					self.connections[conid] = str(connection_starts[i]) + "-" + str(connection_stops[i])

					if len(self.cores[conid]) == 0:
						if retry >= self.max_retry:
							self.cores.pop(conid, None)
							if (self.verbose):
								print("Could not generate connection from " + str(connection_starts[i]) + " to " + str(
									connection_stops[i]) + " as they were too close")
							break
						else:
							retry += 1
					else:
						break
				if retry >= self.max_retry:
					self.cores.pop(conid, None)
					if (self.verbose):
						print("Could not generate connection from " + str(connection_starts[i]) + " to " + str(
							connection_stops[i]))
					break
				else:
					retry += 1

	def generate_cluster(self, cluid):
		"""
		generate cluster
		:param cluid: cluster id
		"""
		retry = 0
		no_space = self.random_start

		#print(self.cores)
		while (True):
			if self.pos[cluid] is not None and retry == 0:
				pos = self.pos[cluid]
				if self.verbose:
					print(f"{cluid}: fixed start {pos}")
			elif cluid == 0 or cluid == 1 or no_space:
				pos = main_generator.random(self.dim) * self.domain_size
				if self.verbose:
					print(f"{cluid}: random start")
			else:
				midids = main_generator.choice(range(cluid), 2, replace=False)
				core1 = self.cores[midids[0]][main_generator.choice(len(self.cores[midids[0]]))]
				core2 = self.cores[midids[1]][main_generator.choice(len(self.cores[midids[1]]))]

				pos = (core1 + core2) / 2
				#print(pos)
				if self.verbose:
					print(f"{cluid}: average start between {midids[0]} and {midids[1]}")
			self.cores[cluid] = []
			noretry = self.generate_cluster_pos(pos, cluid)
			if noretry:
				break
			elif (retry >= self.max_retry and no_space):  # expand domain size, at some point there is enough space for new cluster
				self.domain_size = self.domain_size + self.r_sphere
				retry = 0
				if (self.verbose):
					print(f"{cluid}: domain size increased")
			elif retry >= self.max_retry and not no_space:
				retry = 0
				no_space = True
				if self.verbose:
					print(f"{cluid}: restart random")
			else:
				if (self.verbose):
					print(f"{cluid}: restart")
				retry += 1
		self.cores[cluid] = np.array(self.cores[cluid])

	def make_connection(self, startid, stopid, conid):
		"""
		make connection between two given clusters with given conid
		:param startid: id of start cluster
		:param stopid: id of target cluster
		:param conid: id of connection (negative)
		:return: whether connection succeeded
		"""
		startcore = self.cores[startid][main_generator.choice(len(self.cores[startid]))]
		stopcore = self.cores[stopid][main_generator.choice(len(self.cores[stopid]))]

		conind = conid * -1 - 2
		# print("conind: " + str(conind))

		coni_momentum = self.con_momentum
		if type(self.con_momentum) is list:
			coni_momentum = self.con_momentum[conind]

		dist = np.sum((stopcore - startcore) ** 2) ** (0.5)

		pos = startcore
		self.cores[conid] = []
		self.cores[conid].append(pos)
		pos_old = pos

		min_dist = (self.con_dens_factors[conind] * self.c_sphere + self.dens_factors[
			stopid] * self.r_sphere) * self.con_min_dist
		retry_some = False
		tries_some = -1
		while (dist > min_dist):
			# print(dist)
			retry_last = True
			tries_last = 0
			while (retry_last or retry_some):

				if (retry_some):
					step_dir_old = None

					if (len(self.cores[conid])) <= 1:
						startcore = self.cores[startid][main_generator.choice(len(self.cores[startid]))]
						pos = startcore
						self.cores[conid] = []
						self.cores[conid].append(pos)
						pos_old = pos
					else:
						old_id = main_generator.choice(len(self.cores[conid]))

						# print(old_id)
						# print(len(self.cores[conid]))

						pos_old = self.cores[conid][old_id]

						self.cores[conid] = self.cores[conid][:old_id]

					tries_some += 1
					retry_some = False

				step_dir = (main_generator.random(self.dim) - 0.5)
				step_dir = step_dir / np.linalg.norm(step_dir)

				guided = stopcore - pos_old

				step_dir = step_dir * (1 - coni_momentum) + (guided * coni_momentum)
				step_dir = step_dir / np.linalg.norm(step_dir)

				c_step_clu = self.c_step * self.con_dens_factors[conind]
				max_rand = 1.5
				min_rand = 2 / 3
				random_stepsize = float(main_generator.normal(1, self.step_spread, 1)[0])
				step_randomness = max(min_rand, min(max_rand, random_stepsize))
				# print(step_randomness)
				pos = pos_old + step_dir * c_step_clu * step_randomness

				dist = np.linalg.norm(stopcore - pos)

				for c in self.cores[stopid]:
					dist = min(dist, np.linalg.norm(c - pos))

				attempts = 0

				if (self.tooclose(pos, conid, exclude=[startid, stopid, conid])):
					# print("too close")
					if (tries_some >= self.max_retry):
						return False
					elif (tries_last >= self.max_retry):
						retry_some = True
						tries_last = 0
					else:
						tries_last += 1
				else:
					self.cores[conid].append(pos)
					pos_old = pos
					step_dir_old = step_dir
					retry_last = False

					for d in range(self.dim):
						self.mins[d] = min(pos[d] - self.c_sphere * self.con_dens_factors[conind], self.mins[d])
						self.maxs[d] = max(pos[d] + self.c_sphere * self.con_dens_factors[conind], self.maxs[d])

		connectors = []
		for c in self.cores[conid]:
			if not (self.tooclose_pair(c, conid, startid) or self.tooclose_pair(c, conid, stopid)):
				connectors.append(c)

		if len(connectors) > 0:
			constart = connectors[0]
			target = startcore

			dist = np.linalg.norm(startcore - constart)

			for c in self.cores[startid]:
				cdist = np.linalg.norm(c - constart)
				if dist > cdist:
					dist = cdist
					target = c

			guided = constart - target
			step_dir = guided / np.linalg.norm(guided)

			pos = target + (self.con_dens_factors[conind] * self.c_sphere + self.dens_factors[
				startid] * self.r_sphere) * step_dir * self.con_min_dist

			connectors = deque(connectors)
			connectors.appendleft(pos)
			connectors = list(connectors)

			constop = connectors[-1]
			target = stopcore

			dist = np.linalg.norm(stopcore - constop)

			for c in self.cores[stopid]:
				cdist = np.linalg.norm(c - constop)
				if dist > cdist:
					dist = cdist
					target = c

			guided = constop - target
			step_dir = guided / np.linalg.norm(guided)

			pos = target + (self.con_dens_factors[conind] * self.c_sphere + self.dens_factors[
				stopid] * self.r_sphere) * step_dir * self.con_min_dist

			connectors.append(pos)

		self.cores[conid] = np.array(connectors)
		return True

	# check if a position is too close to exitsing cores
	def tooclose(self, pos, label, exclude=None, noise=False):
		"""
		perform a check regarding closeness
		:param pos: position
		:param label: own label
		:param exclude: labels to exclude
		:param noise: whether pos is intended for noise or a core
		:return: whether too close
		"""
		# if (len(points) > 0):
		#    points = points[0]

		#print(self.cores)

		if exclude is None:
			exclude = [label]

		for cluid in self.cores.keys():
			if cluid not in exclude:
				# print(cluid)
				if cluid < -1:
					factor = self.con_dens_factors[cluid * -1 - 2] * self.c_sphere
				else:
					factor = self.dens_factors[cluid] * self.r_sphere

				if label < -1:
					factor += self.con_dens_factors[label * -1 - 2] * self.c_sphere
				elif not noise:
					factor += self.dens_factors[label] * self.r_sphere
				elif self.safety:
					factor += factor
				else:
					factor += 0.1 * factor

				if label < -1:
					min_distcluid = self.con_min_dist * factor
				else:
					min_distcluid = self.min_dist * factor
				# print(min_distcluid)
				for core in self.cores[cluid]:
					dist = 0
					for i in range(self.dim):
						dist += (core[i] - pos[i]) ** 2
						if dist ** 0.5 > min_distcluid:
							break
					if dist ** 0.5 < min_distcluid:
						return True
		return False

	def tooclose_pair(self, pos, label, partner):
		"""
		perform pairwise check regarding closeness
		:param pos: position
		:param label: own label
		:param partner: label of partner
		:return: whether too close
		"""
		factor1 = 0
		if label >= 0:
			factor1 = self.dens_factors[label] * self.r_sphere
		else:
			factor1 = self.con_dens_factors[label * -1 - 2] * self.c_sphere
		factor2 = 0
		if partner >= 0:
			factor2 = self.dens_factors[partner] * self.r_sphere
		else:
			factor2 = self.con_dens_factors[partner * -1 - 2] * self.c_sphere

		factor = factor1 + factor2

		if label < -1:
			min_distcluid = self.con_min_dist * factor
		else:
			min_distcluid = self.min_dist * factor
		if label < 0 or partner < 0:
			min_distcluid = self.con_min_dist * factor

			# print(str(factor1) + " " + str(factor2) + " " + str(factor) + " " + str(min_distcluid))
		for core in self.cores[partner]:
			dist = 0
			for i in range(self.dim):
				dist += (core[i] - pos[i]) ** 2
				if dist ** 0.5 > min_distcluid:
					break
			if dist ** 0.5 < min_distcluid:
				return True
		return False

	def generate_cluster_pos(self, start_pos, cluid):
		"""
		generate a cluster with given id at a specific location
		:param start_pos: cluster start location
		:param cluid: cluster id
		:return: whether cluster generation succeeded
		"""
		if self.tooclose(start_pos, cluid):
			return False

		clu_momentum = self.momentum
		if type(self.momentum) is list:
			clu_momentum = self.momentum[cluid]

		core_num = 0
		if type(self.core_num) is list:
			core_num = self.core_num[cluid]
		else:
			clu_ratio = 0
			if cluid >= 1:
				clu_ratio = self.clu_ratios[cluid] - self.clu_ratios[cluid - 1]
			else:
				clu_ratio = self.clu_ratios[cluid]
			core_num = max(round((clu_ratio) * self.core_num / self.data_ratio), 1)

		if (self.verbose):
			print("Cluster ID " + str(cluid) + " Core Number: " + str(core_num))
		pos = start_pos
		self.cores[cluid].append(pos)
		pos_old = pos
		step_dir_old = None

		branch_chance = self.branch[cluid]
		star_chance = self.star[cluid]

		for i in range(core_num - 1):

			retry_last = True
			tries_last = 0
			retry_some = False
			tries_some = -1
			while (retry_last or retry_some):
				rand = main_generator.random()
				if (retry_some or rand < branch_chance):
					step_dir_old = None
					probs = [1 / len(self.cores[cluid])] * len(self.cores[cluid])
					if star_chance > 0 and len(self.cores[cluid]) > 1:
						probs = [star_chance]
						probs.extend([(1 - star_chance) / (len(self.cores[cluid]) - 1)] * (len(self.cores[cluid]) - 1))

					selected = main_generator.choice(len(self.cores[cluid]), p=probs)
					# print(selected)
					pos_old = self.cores[cluid][selected]
					tries_some += 1
					retry_some = False

				step_dir = (main_generator.random(self.dim) - 0.5)
				step_dir = step_dir / np.linalg.norm(step_dir)
				if step_dir_old is not None:
					step_dir = step_dir * (1 - clu_momentum) + (step_dir_old * clu_momentum)
				step_dir = step_dir / np.linalg.norm(step_dir)
				# print(step_dir)

				r_step_clu = self.r_step * self.dens_factors[cluid]
				max_rand = 1.5
				min_rand = 2 / 3
				random_stepsize = float(main_generator.normal(1, self.step_spread, 1)[0])
				step_randomness = max(min_rand, min(max_rand, random_stepsize))
				# print(step_randomness)
				pos = pos_old + step_dir * r_step_clu * step_randomness
				attempts = 0

				if (self.tooclose(pos, cluid)):
					if (tries_some >= self.max_retry):
						return False
					elif (tries_last >= self.max_retry):
						retry_some = True
						tries_last = 0
					else:
						tries_last += 1
				else:
					self.cores[cluid].append(pos)
					pos_old = pos
					step_dir_old = step_dir
					retry_last = False

					for d in range(self.dim):
						self.mins[d] = min(pos[d] - self.r_sphere * self.dens_factors[cluid], self.mins[d])
						self.maxs[d] = max(pos[d] + self.r_sphere * self.dens_factors[cluid], self.maxs[d])

		return True


	def generate_data(self, data_num, center=False, non_zero=False, seed=None, equal=False):
		"""
		Generate data points based on underlying skeleton
		:param data_num: number of data points
		:param center: whether to place a data point at the core center
		:param non_zero: whether cores should be guaranteed to have at least one data point
		:param seed: new seed for data generation
		:param: equal: spread out data points as equal as possible across cluster
		:return: data points as  np.array of shape (dim+1, data_num), last column is cluster id
		"""
		testsum = 0
		mins = []
		maxs = []
		data = []

		if seed is not None:
			set_data_seed(seed)
		else:
			set_data_seed(self.seed)


		con_core_num = 0
		for cluid in self.cores.keys():
			if cluid < -1:
				con_core_num += len(self.cores[cluid])

		for cluid in self.cores.keys():
			if cluid >= 1:
				dist = self.distributions[cluid]
				cluradius = self.r_sphere * self.dens_factors[cluid]
				clu_ratio = self.clu_ratios[cluid] - self.clu_ratios[cluid - 1]
			elif cluid == 0:
				dist = self.distributions[cluid]
				cluradius = self.r_sphere * self.dens_factors[cluid]
				clu_ratio = self.clu_ratios[cluid]
			else:
				dist = self.con_distribution
				clu_ratio = self.ratio_con * len(self.cores[cluid]) / con_core_num
				cluradius = self.c_sphere * self.con_dens_factors[-1 * cluid - 2]
			clu_core_num = len(self.cores[cluid])

			# print(cluid)
			clu_data_num = round(clu_ratio * data_num)
			testsum += clu_data_num
			if equal:
				spread_val = clu_data_num//clu_core_num
				assignment = np_data_generator.choice(clu_core_num, clu_data_num%clu_core_num, replace=False)
				assignment_counter = Counter(assignment)
			else:
				spread_val = 0
				assignment = np_data_generator.choice(clu_core_num, clu_data_num)
				assignment_counter = Counter(assignment)
			for coreid in range(len(self.cores[cluid])):
				core = self.cores[cluid][coreid]
				core_data_num = assignment_counter[coreid] + spread_val
				if core_data_num == 0 and non_zero:
					core_data_num = 1
				if center and core_data_num > 0:
					core_data_num -= 1
					data.append(core.copy().tolist() + [cluid])
				# print(str(coreid) + " " + str(core_data_num))
				if core_data_num > 0:
					data_new = self.generate_distribution(core, cluradius, core_data_num, dist, cluid)
					data.extend(data_new.tolist())
		if data_num - len(data) > 0 and self.ratio_noise == 0:
			clus = np_data_generator.choice(self.clunum, data_num - len(data), replace=True)
			for cluid in clus:
				clu_core_num = len(self.cores[cluid])
				dist = self.distributions[cluid]
				cluradius = self.r_sphere * self.dens_factors[cluid]
				coreid = np_data_generator.choice(clu_core_num,1)
				core = self.cores[cluid][coreid]
				data_new = self.generate_distribution(core, cluradius, 1, dist, cluid)
				data.extend(data_new.tolist())

		noisenum = max(round(data_num * self.ratio_noise), data_num - len(data))
		#print("diff:", data_num - len(data), "noisenum:", noisenum, flush=True)
		if self.ratio_noise == 0:
			noisenum = 0
		noise = np_data_generator.random([noisenum, self.dim + 1])

		maxall = -1 * np.inf
		minall = np.inf
		if self.square:
			for d in range(self.dim):
				if maxall < self.maxs[d]:
					maxall = self.maxs[d]
				if minall > self.mins[d]:
					minall = self.mins[d]
			dspan = maxall - minall
			for d in range(self.dim):
				noise[:, d] = (noise[:, d] * dspan * 1.2) + minall - 0.1 * dspan
		else:
			for d in range(self.dim):
				dspan = self.maxs[d] - self.mins[d]

				noise[:, d] = (noise[:, d] * dspan * 1.2) + self.mins[d] - 0.1 * dspan

		noise[:, -1] = -1
		truenoise = []
		for n in noise:
			while (self.tooclose(n, -1, noise=True)):
				n = np_data_generator.random([self.dim + 1])
				for d in range(self.dim):
					dspan = self.maxs[d] - self.mins[d]
					n[d] = (n[d] * dspan * 1.2) + self.mins[d] - 0.1 * dspan
					n[-1] = -1
			truenoise.append(n)

			# print("noise removed")

		data.extend(truenoise)
		# print(len(data))

		if not non_zero:
			while (len(data) > data_num):
				data.pop()
				print(len(data))

		return np.array(data)

	def generate_distribution(self, core, cluradius, core_data_num, dist, cluid):
		if dist == "uniform":
			return random_ball_num(core, cluradius, self.dim, core_data_num, cluid)
		elif dist == "paper":
			return random_ball_num_bias(core, cluradius, self.dim, core_data_num, cluid)
		elif dist == "gaussian":
			gseed = round(np_data_generator.random()*1000000)
			return gaussian_ball_num(core, cluradius, self.dim, core_data_num, cluid, gseed)

		else:
			tseed = round(np_data_generator.random()*1000000)
			return t_ball_num(core, cluradius, self.dim, core_data_num, cluid, float(self.distributions[cluid]), tseed)

	def paint(self, dim1, dim2, data=None, show_radius=True, show_core=True, cores=None):
		"""
		Data Generator Painter
		:param dim1: first dimension of plot
		:param dim2: second dimension of plot
		:param data: data points to draw
		:param show_radius: whether to show the core radii
		:param show_core: whether to show the core positions
		:param cores: if only specific core are supposed to be drawn
		"""
		if cores is None:
			cores = self.cores

		num_col = max(self.cores.keys()) - min(0, min(self.cores.keys())) + 2
		color = plt.cm.tab20(np.linspace(0, 1, num_col))
		plt.figure(figsize=(15, 15))
		if data is not None:
			plt.scatter(data[:, dim1], data[:, dim2], color=color[data[:, len(data[0]) - 1].astype('int32') + 1],
						alpha=1)

		legend = []

		for cluid in cores.keys():
			if len(cores[cluid]) > 0:
				if self.verbose:
					if cluid >= 0:
						patch = mpatches.Patch(color=color[cluid + 1], label=cluid)
						legend.append(patch)
					else:
						patch = mpatches.Patch(color=color[cluid + 1], label=self.connections[cluid])
						legend.append(patch)
				# print(self.cores[cluid])
				if show_core:
					if data is None:
						plt.scatter(cores[cluid][:, dim1], cores[cluid][:, dim2], color=color[cluid + 1], alpha=1)
					else:
						plt.scatter(cores[cluid][:, dim1], cores[cluid][:, dim2], color='black', alpha=1)
				if show_radius:
					for core in cores[cluid]:
						# print(core[0])
						if cluid >= 0:
							plt.gca().add_patch(mpatches.Circle((core[dim1], core[dim2]),
																radius=self.r_sphere * self.dens_factors[cluid],
																color=color[cluid + 1], alpha=0.2))
						else:
							plt.gca().add_patch(mpatches.Circle((core[dim1], core[dim2]),
																radius=self.c_sphere * self.con_dens_factors[
																	(-1 * cluid) - 2], color=color[cluid + 1],
																alpha=0.2))

		if data is not None:
			if -1 in data[:, len(data[0]) - 1]:
				patch = mpatches.Patch(color=color[0], label="Noise")
				legend.append(patch)

		plt.axis('scaled')

		plt.ylabel(dim2 + 1)
		plt.xlabel(dim1 + 1)
		if self.verbose:
			plt.legend(handles=legend)
		plt.show()

	def display_data(self, data, show_radius=False, show_core=False, dims=None, dcount=2):
		"""
		Display data set
		:param data: data set
		:param show_radius: whether to show the core radii
		:param show_core: whether to show the core positions
		:param dims: specific dimensions to show (if there are more than 2)
		:param dcount: amount of dimensions to display pairs of (if no specific dimensions are given)
		"""
		if (self.dim == 2):
			self.paint(0, 1, data=data, show_radius=show_radius, show_core=show_core)
		elif dims is not None:
			for d1 in dims:
				for d2 in dims:
					if d1 < d2:
						self.paint(d1, d2, data=data, show_radius=show_radius, show_core=show_core)
		else:
			diff = []
			for d in range(self.dim):
				diff.append(self.maxs[d] - self.mins[d])

			# print(diff)
			dims = np.argsort(diff)[:dcount]
			# print(dims)
			for d1 in dims:
				for d2 in dims:
					if d1 < d2:
						self.paint(d1, d2, data=data, show_radius=show_radius, show_core=show_core)

	def display_cores(self, dims=None, dcount=2):
		"""
		Display data generator skeleton
		:param dims: specific dimensions to show (if there are more than 2)
		:param dcount: amount of dimensions to display pairs of (if no specific dimensions are given)
		"""
		# print(self.cores)
		if (self.dim == 2):
			self.paint(0, 1)
		elif dims is not None:
			for d1 in dims:
				for d2 in dims:
					if d1 < d2:
						self.paint(d1, d2)
		else:
			diff = []
			for d in range(self.dim):
				diff.append(self.maxs[d] - self.mins[d])

			# print(diff)
			dims = np.argsort(diff)[:dcount]
			# print(dims)
			for d1 in dims:
				for d2 in dims:
					if d1 < d2:
						self.paint(d1, d2)

	def display_cores_selected(self, cores, dims=None, dcount=2):
		"""
		Display parts of the data generator skeleton
		:param cores: which cores to display
		:param dims: specific dimensions to show (if there are more than 2)
		:param dcount: amount of dimensions to display pairs of (if no specific dimensions are given)
		"""
		if (self.dim == 2):
			self.paint(0, 1, cores=cores)
		elif dims is not None:
			for d1 in dims:
				for d2 in dims:
					if d1 < d2:
						self.paint(d1, d2, cores=cores)
		else:
			diff = []
			for d in range(self.dim):
				diff.append(self.maxs[d] - self.mins[d])

			# print(diff)
			dims = np.argsort(diff)[:dcount]
			# print(dims)
			for d1 in dims:
				for d2 in dims:
					if d1 < d2:
						self.paint(d1, d2, cores=cores)

	def toggle_verbose(self):
		"""
		swaps verbose setting
		"""
		self.verbose = not self.verbose

	def init_stream(self, command="", default_duration=1000):
		"""
		Initialize stream based on strong command
		:param command: String command for stream
		:param default_duration: default duration of a stream block
		:return: stream settings:  stream content, which part repeats, data_cores_block, num_block, noise_block
		"""
		# command_repeats = nestedExpr('[',']').parseString('[' + str(command) + ']').asList()

		command = command + " "
		block = 0
		loopcount = 0
		loopstart = {}
		loopstop = {}

		curLoopId = 0

		data_cores_block = {}
		data_cores_block[0] = {}
		num = {}

		noise_block = {0: 0}
		num_block = {}

		nested_loop = {}

		loopstart[0] = 0

		i = 0
		while i < len(command):
			# print(i)
			# print(command[i])

			if command[i] == "(":
				curLoopId += 1
				loopstart[curLoopId] = block
				# print("start seen " + str(curLoopId))

			elif command[i] == ")":

				curLoopStopId = len(loopstart) - 1
				while curLoopStopId in loopstop.keys():
					curLoopStopId -= 1

				loopstop[curLoopStopId] = block
				if i < len(command) - 1:
					if command[i + 1] == "{":
						j = 2
						num_loop = ""
						while command[i + j] != "}":
							num_loop = num_loop + command[i + j]
							j += 1
						num_loop = int(num_loop)
						i = i + j
						num[curLoopStopId] = num_loop
					else:
						num[curLoopStopId] = 'x'
				else:
					num[curLoopStopId] = 'x'
			elif command[i] == "|":
				if command[i + 1] == "{":
					j = 2
					c_block = ""
					while command[i + j] != "}":
						c_block = c_block + command[i + j]
						j += 1
					c_block = int(c_block)
					i = i + j
					num_block[block] = c_block
				else:
					num_block[block] = default_duration

				block += 1
				data_cores_block[block] = {}
				noise_block[block] = 0

			elif command[i] == "n":
				noise_block[block] = 1

			elif command[i].isnumeric() or command[i] == "-":
				cid = command[i]
				if command[i] == "-":
					cid += command[i + 1]
					i += 1
				while command[i + 1].isnumeric():
					cid = cid + command[i]
					i += 1
				cid = int(cid)
				if command[i + 1] == "c":
					i += 2
					eid = command[i]
					while command[i + 1].isnumeric():
						eid = eid + command[i]
						i += 1
					eid = int(eid)

					for conkey in self.connections.keys():
						if (self.connections[conkey] == str(cid) + "-" + str(eid) or self.connections[conkey] == str(
								eid) + "-" + str(cid)):
							cid = conkey


				if (command[i + 1] == "["):
					j = 2
					# print("I was here")
					# print(block)
					# print(cid)

					# print(data_cores_block[block])
					data_cores_block[block][cid] = []
					# print(data_cores_block[block][cid])

					while command[i + j] != "]":
						if command[i + j] != ",":
							cstartid = command[i + j]
							while command[i + j + 1].isnumeric():
								cstartid = cstartid + command[i + j + 1]
								j += 1
							cstartid = int(cstartid)

							if command[i + j + 1] == ":":
								j += 2
								cstopid = command[i + j]

								while command[i + j + 1].isnumeric():
									cstopid = cstopid + command[i + j + 1]
									j += 1
								cstopid = int(cstopid)
								data_cores_block[block][cid].extend(np.arange(cstartid, cstopid + 1).tolist())
							else:
								data_cores_block[block][cid].append(cstartid)
						j += 1
					i = i + j
				else:
					data_cores_block[block][cid] = np.arange(len(self.cores[cid])).tolist()
			i += 1

		loopstop[0] = block
		num[0] = 1

		if block not in num_block.keys():
			num_block[block] = default_duration

		for l in range(len(loopstart)):
			startl = loopstart[l]
			stopl = loopstop[l]
			l2 = l + 1
			nested_loop[l] = []
			while l2 in range(len(loopstart)) and loopstart[l2] <= stopl:
				nested_loop[l].append(l2)
				l2 += 1
		for l in nested_loop.keys():
			prune = []
			for l2 in nested_loop[l]:
				for l3 in nested_loop[l]:
					if l3 in nested_loop[l2]:
						prune.append(l3)
			update = []
			for l2 in nested_loop[l]:
				if l2 not in prune:
					update.append(l2)
			nested_loop[l] = update

		content = {}
		remaining_loops = len(nested_loop)

		for i in num.keys():
			if len(num.keys()) < 2:
				break
			elif i != list(num.keys())[-2]:
				if num[i] == 'x':
					num[i] = 2
			else:
				if num[i] == 'x':
					if loopstop[i] != block:
						num[0] = 'x'
						num[i] = 2

		while remaining_loops > 0:

			for lk in nested_loop.keys():
				if len(nested_loop[lk]) > 0 and not lk in content.keys():
					wait = False
					for lk2 in nested_loop[lk]:
						if not lk2 in content.keys():
							wait = True
					if not wait:
						startb = loopstart[lk]
						stopb = loopstop[lk]
						content[lk] = []
						for lk2 in nested_loop[lk]:
							repeats = 1
							content[lk].extend(np.arange(startb, loopstart[lk2]).tolist())
							if num[lk2] != 'x':
								repeats = num[lk2]
							for j in range(repeats):
								content[lk].extend(content[lk2])
							startb = loopstop[lk2] + 1
						remaining_loops -= 1
						if startb < stopb + 1:
							content[lk].extend(np.arange(startb, stopb + 1).tolist())

						# print(str(lk) + " done: " + str(remaining_loops))
				else:
					if not lk in content.keys():
						content[lk] = np.arange(loopstart[lk], loopstop[lk] + 1).tolist()
						remaining_loops -= 1
						# print(str(lk) + " done: " + str(remaining_loops))

		# print(content)

		repeat = ""
		if not len(num.keys()) < 2:
			finalloop = list(num.keys())[-2]
			if num[finalloop] == 'x':
				repeat = content[finalloop]
			elif loopstop[finalloop] != block:
				repeat = [block]
			else:
				repeat = content[0]
		else:
			repeat = [block]

		# print(final)

		self.stream_content = content[0]
		self.stream_repeat = repeat
		self.stream_data_cores_block = data_cores_block
		self.stream_num_block = num_block
		self.noise_block = noise_block

		self.cur_stream_point = 0
		self.cur_stream_pos = 0
		# self.prev_block_num = 0
		self.in_repeat = False
		self.cur_block_num = 0
		self.cur_stream_dist = {}
		if len(num_block) > 0:
			self.cur_block_num = num_block[0]
			self.cur_stream_dist = {}

		return content[0], repeat, data_cores_block, num_block, noise_block

	def display_stream(self, command="", default_duration=1000, show_core=True, show_radius=True):
		"""
		Display the behaviour of given stream command String (also sets the command to be the current stream)
		:param command: String command for stream
		:param default_duration: default duration of a stream block
		:param show_radius: whether to show the core radii
		:param show_core: whether to show the core positions
		"""
		self.init_stream(command=command, default_duration=default_duration)
		self.display_current_stream(show_core, show_radius)

	def display_current_stream(self, show_core=True, show_radius=True):
		"""
		Display the behaviour of the currently set stream command
		:param show_radius: whether to show the core radii
		:param show_core: whether to show the core positions
		"""
		points = 0

		for block in self.stream_content:
			# print(block)
			# print(data_cores_block[block])
			# print(num_block[block])

			block_cores = {}
			for cluid in self.stream_data_cores_block[block].keys():
				block_cores[cluid] = self.cores[cluid][self.stream_data_cores_block[block][cluid]]
			self.display_cores_selected(block_cores,show_core,show_radius)

	def __iter__(self):
		"""
		:return: return stream iterator object (itself)
		"""
		return self

	def __next__(self):
		"""
		:return: next data point based on current stream state
		"""
		recalc_db = False

		if self.cur_stream_point == 0 and self.cur_stream_pos == 0:
			recalc_db = True

		self.cur_stream_point += 1

		if not self.in_repeat:
			block = self.stream_content[self.cur_stream_pos]

			if self.cur_stream_point > self.cur_block_num:
				self.cur_stream_pos += 1
				self.cur_stream_point = 1

				if self.cur_stream_pos >= len(self.stream_content):
					self.in_repeat = True
					self.cur_stream_pos = 0
					block = self.stream_repeat[self.cur_stream_pos]
				else:
					block = self.stream_content[self.cur_stream_pos]
				self.cur_block_num = self.stream_num_block[block]
				recalc_db = True
		else:
			block = self.stream_repeat[self.cur_stream_pos]
			if self.cur_stream_point > self.cur_block_num:
				self.cur_stream_pos += 1
				self.cur_stream_point = 1

				if self.cur_stream_pos >= len(self.stream_repeat):
					self.cur_stream_pos = 0
				block = self.stream_repeat[self.cur_stream_pos]
				self.cur_block_num = self.stream_num_block[block]
				recalc_db = True
		if recalc_db:
			block_db = {}

			con_core_num = 0
			for cluid in self.stream_data_cores_block[block].keys():
				if cluid < -1:
					con_core_num += len(self.stream_data_cores_block[block][cluid])

			db_sum = 0
			for cluid in self.stream_data_cores_block[block].keys():
				if cluid >= 1:
					block_db[cluid] = self.clu_ratios[cluid] - self.clu_ratios[cluid - 1]
				elif cluid == 0:
					block_db[cluid] = self.clu_ratios[cluid]
				else:
					block_db[cluid] = self.ratio_con * len(self.stream_data_cores_block[block][cluid]) / con_core_num
				db_sum += block_db[cluid]

			for cluid in block_db.keys():
				block_db[cluid] = (block_db[cluid] / db_sum) * (1 - self.noise_block[block] * self.ratio_noise)

			if self.noise_block[block] == 1:
				block_db[-1] = self.ratio_noise
			self.cur_stream_db = block_db

		cluid_choice = np_data_generator.choice(list(self.cur_stream_db.keys()), 1, p=list(self.cur_stream_db.values()))[0]

		# self.cores[cluid][self.stream_data_cores_block[block][cluid]]
		if cluid_choice == -1:
			n = np_data_generator.random([self.dim + 1])
			for d in range(self.dim):
				dspan = self.maxs[d] - self.mins[d]
				n[d] = (n[d] * dspan * 1.2) + self.mins[d] - 0.1 * dspan
				n[-1] = -1
			while (self.tooclose(n, -1, noise=True)):
				n = np_data_generator.random([self.dim + 1])
				for d in range(self.dim):
					dspan = self.maxs[d] - self.mins[d]
					n[d] = (n[d] * dspan * 1.2) + self.mins[d] - 0.1 * dspan
					n[-1] = -1
			return n
		else:
			core_choice = np_data_generator.choice(self.stream_data_cores_block[block][cluid_choice])
			core_chosen_pos = self.cores[cluid_choice][core_choice]
			cluradius = 0
			if cluid_choice >= 0:
				dist = self.distributions[cluid_choice]
				cluradius = self.r_sphere * self.dens_factors[cluid_choice]
			else:
				cluradius = self.c_sphere * self.con_dens_factors[-1 * cluid_choice - 2]
				dist = "uniform"
			return self.generate_distribution(core_chosen_pos, cluradius, 1, dist, cluid_choice)[0]