update comments for python source codes

		batch_id (int): id of the batch

	Returns: 

		project site with all batches at ``/annotator/<p_name>``, if the annotation is saved or the valid annotation is submitted.

		project site with all batches at ``/annotator/<p_name>``, 

		if the annotation is saved or the valid annotation is submitted.

	Error:

		Error message emerges if invalid annotation is submitted.

		# Submit Button

		if request.form['action'] == 'submit':

			# in case this person has opened this batch more than once and already submitted but still 

			# wants to submit this batch again

			# in case this person has opened this batch more than once and already submitted 

			# but still wants to submit this batch again

			if current_batch in current_annotator.batches:

				flash(f'Batch {batch_id} already submitted!', 'action')

						data.worst_id = form.worst_item.data

					else:

						Data(best_id=form.best_item.data, worst_id=form.worst_item.data, tuple_=tuple_, annotator=current_annotator)

						Data(best_id=form.best_item.data, worst_id=form.worst_item.data, 

								tuple_=tuple_, annotator=current_annotator)

				current_annotator.batches.append(current_batch)

				db.session.commit()

				flash(f'Batch {batch_id} successfully submitted!', 'action')

					data.worst_id = form.worst_item.data

				else:

					Data(best_id=form.best_item.data, worst_id=form.worst_item.data, tuple_=tuple_, annotator=current_annotator)

					Data(best_id=form.best_item.data, worst_id=form.worst_item.data, 

							tuple_=tuple_, annotator=current_annotator)

			db.session.commit()

			flash(f'Batch {batch_id} saved!', 'action')

	return render_template('annotator/batch.html', best=best, worst=worst, description=description, 

							 zip=zip, batch_form=forms,  question_number=question_number, 

							  tuples=tuples, p_name=p_name, batch_id=batch_id, int=int, tuple_size=len(tuple_.items))

							tuples=tuples, p_name=p_name, batch_id=batch_id, int=int, 

							tuple_size=len(tuple_.items))

# MTurk - A batch of a project

class TupleForm(FlaskForm):

	"""

	Extend :fform:`FlaskForm <flask_wtf.FlaskForm>`. Define form for a tuple with two choices for **best** and **worst** item.

	Extend :fform:`FlaskForm <flask_wtf.FlaskForm>`.

	Define form for a tuple with two choices for **best** and **worst** item.

	Attributes:

		best_item (:field:`RadioField <wtforms.fields.RadioField>`): answer for the question of best item

		worst_item (:field:`RadioField <wtforms.fields.RadioField>`): answer for the question of worst item

		best_item (:field:`RadioField <wtforms.fields.RadioField>`): answer for the question

																	of best item

		worst_item (:field:`RadioField <wtforms.fields.RadioField>`): answer for the question

																	of worst item

	"""

	best_item = RadioField('Label', coerce=int, 

					validators= [InputRequired(message=u'Choose one item!'), \

										NotEqualTo('worst_item', message=u'Two questions for this tuple require 2 different answers')])

								NotEqualTo('worst_item', 

								message=u'Two questions for this tuple require 2 different answers')

								])

	worst_item = RadioField('Label', coerce=int, 

					validators= [InputRequired(message=u'Choose one item!'),

										NotEqualTo('best_item', message=u'Two questions for this tuple require 2 different answers')])

								NotEqualTo('best_item', 

								message=u'Two questions for this tuple require 2 different answers')

								])

class AnnoCheckinForm(FlaskForm):

	"""

	keyword = PasswordField('Keyword as annotator', 

							validators=[InputRequired(message=u''), 

										InputValid(model=Annotator, field=Annotator.keyword, message=u'Invalid keyword!')])

										InputValid(model=Annotator, field=Annotator.keyword, 

											message=u'Invalid keyword!')

										])

	name = StringField('Specify your name', validators=[InputRequired(message=u'')])

		"""

		Override  :form:`validate() <wtforms.form.Form.validate>`.

		Check if the person using this keyword is the first one who logged in by checking his given (pseudo)name.

		Check if the person using this keyword is the first one who logged in

		by checking his given (pseudo)name.

		"""

		if not FlaskForm.validate(self):

			return False

		if Annotator.query.filter(Annotator.keyword==self.keyword.data, Annotator.name.isnot(None), Annotator.name==self.name.data).first() \

		 or Annotator.query.filter(Annotator.keyword==self.keyword.data, Annotator.name==None).first():

			return True

		self.name.errors.append(u'This keyword is used, account is no longer available!')

		self.name.errors.append(u'This name is not used for this keyword! Have you logged in with this keyword before? If not, someone has already used it!')

		return False

*Module* ``project.annotator.helpers``

This module defines some helper functions to deal with problems of each subroute inside Annotator-System.

This module defines some helper functions to deal with problems of each subroute

inside Annotator-System.

"""

		else:

			check_batches.append(False)

	return render_template('annotator/project.html', batches_links=batches_list(p_name, n_batches=len(current_project.batches)),

	return render_template('annotator/project.html', 

							batches_links=batches_list(p_name, n_batches=len(current_project.batches)),

							zip=zip, check=check_batches, project=current_project)

"""

*Module* ``project.generator``

This module provides some classes on purpose of generating and handling different types of datas 

during running the application. This can be used outside the application as it works independently.

This module provides some classes on purpose of generating and handling

different types of datas during running the application. This can be used

outside the application as it works independently.

"""

from collections import Counter

from random import shuffle, choice

from itertools import product, chain

from sys import stderr, stdout, exit

class BaseGenerator(object):

	"""

	This class contains a base function :meth:`get_frequency` to count occurencies of each items in all tuples.

	This class contains a base function :meth:`get_frequency` to count

	occurencies of each items in all tuples.

	Args:

		tuples (list(tuple or set or list)): list of tuples/lists/sets of items

	Attributes:

		frequencies (collections.Counter): container in form of a dictionary to save all items and frequencies inside given lists of sets 

		frequencies (collections.Counter): container in form of a dictionary to

				save all items and frequencies inside given lists of sets

	Examples:

		>>> tuples = [('A','B','C'), ('B','C','D'),('D','A','C')]

class DataGenerator(BaseGenerator):

	"""

	Extend :class:`BaseGenerator`. Create an object of input datas for the survey based on input file(s).

	Extend :class:`BaseGenerator`. Create an object of input datas for the

	survey based on input file(s).

	Args:

		num_iter (int, optional): number of necessary iterations to generate tuples, *default:* ``100``

		num_iter (int, optional): number of necessary iterations to generate 

					tuples, *default:* ``100``

		batch_size (int, optional): size of a normal batch, *default:* ``20``

		minimum (int, optional): minimum size of a batch to be formed if the rest items do not meet the normal size, *default:* ``5``

		minimum (int, optional): minimum size of a batch to be formed if the

					rest items do not meet the normal size, *default:* ``5``

	Attributes:

		items (set): the unique given items

		tuples (list): list of all unique generated tuples with the best results after all iterations

		tuples (list): list of all unique generated tuples with the best

				results after all iterations

		batches (dict): all batches prepared for questionnaire

		num_iter (int): number of necessary iterations to generate tuples, *default:* ``100``

		num_iter (int): number of necessary iterations to generate tuples,

				*default:* ``100``

		batch_size (int): size of a normal batch, *default:* ``20``

		minimum (int): minimum size of a batch to be formed if the rest items do not meet the normal :attr:`batch_size`, *default:* ``5``

		factor (int or float): to decide the number of tuples to be generated - `n_tuples =` :attr:`factor` `* len(` :attr:`items` `)`, *default:* ``2`` if fewer than 10000 items 

		tuple_size (int): size of each tuple, *default:* ``4`` if fewer than 1000 items else *5*

		minimum (int): minimum size of a batch to be formed if the rest

				items do not meet the normal :attr:`batch_size`, *default:* ``5``

		factor (int or float): to decide the number of tuples to be 

				generated - `n_tuples =` :attr:`factor` `* len(` :attr:`items` `)`,

				*default:* ``2`` if fewer than 10000 items 

		tuple_size (int): size of each tuple, *default:* ``4`` if fewer than 1000

				items else ``5``

	Examples:

		>>> example = open('../examples/movie_reviews_examples.txt','rb')

	def generate_tuples(self):

		"""

		Generate tuples, this is a reimplementation of `generate-BWS-tuples.pl` in :bws:`source code <Best-Worst-Scaling-Scripts.zip>`.

		Generate tuples, this is a reimplementation of `generate-BWS-tuples.pl`

		in :bws:`source code <Best-Worst-Scaling-Scripts.zip>`.

		The tuples are generated by random sampling and satisfy the following criteria:

		The tuples are generated by random sampling and satisfy the following

		criteria:

			1. no two items within a tuple are identical; 

			2. each item in the item list appears approximately in the same number of tuples; 

			3. each pair of items appears approximately in the same number of tuples.

		Returns:

			list: update list of all unique generated tuples with the best results after all (attribute :attr:`tuples`).

			list: update list of all unique generated tuples with the best results

			after all (attribute :attr:`tuples`).

		Raises:

			ValueError: if the number of :attr:`items` is fewer than :attr:`tuple_size`.

		num_items = len(items)

		# check if the number of unique items is not less than the number of items requested per tuple

		# check if the number of unique items is not less than 

		# the number of items requested per tuple

		if num_items < self.tuple_size:

			raise ValueError('The number of unique items is less than the number of items requested per tuple')

			raise ValueError('''The number of unique items is less than the number 

										of items requested per tuple''')

		# generate tuples

		number_tuples = int(0.5 + self.factor * num_items)

				# check if there are enough remained items in the random list for a new tuple

				if (curr_ind + self.tuple_size) <= len(random_items):

					# set a new tuple with tuple_size items in the random list starting at index curr_ind

					# set a new tuple with tuple_size items in the random list 

					# starting at index curr_ind

					new_tuple = set(random_items[curr_ind:curr_ind+self.tuple_size])

					curr_ind += self.tuple_size

			if i_iter == 0 or score < best_score:

				best_score = score

				best_tuples = tuples[:]

				iter_ = i_iter

				# iter_ = i_iter

		# print('Choose from iteration {} with score {}'.format(iter_+1, best_score))

		Split whole set of tuples into batches.

		Returns:

			dict(int = list): update all batches prepared for questionnaire (attribute :attr:`batches`).

			dict(int = list): update all batches prepared for questionnaire

					(attribute :attr:`batches`).

		Raises:

			ValueError: if there is no attribute :attr:`tuples`.

	def generate_data(self):

		"""

		Generate datas including tuples and batches. This method calls :meth:`generate_tuples` and :meth:`generate_batches`.

		Generate datas including tuples and batches. This method calls

		:meth:`generate_tuples` and :meth:`generate_batches`.

		"""

		self.generate_tuples()

		self.generate_batches()

		Calculate scores of the items using formula of :orme09:`Orme 2009 <indivmaxdiff.pdf>`.

		Returns:

			list(tuple(str, float)): descendingly sorted list of tuples ``(item, score)`` based on scores

			list(tuple(str, float)): descendingly sorted list of tuples ``(item, score)``

			based on scores

		References:

			More about research with :bws:`Best-Worst-Scaling <>`

			else:

				scores[item] = (pos - neg) / self.frequencies[item]

		return sorted(scores.items(), key=lambda item: item[1], reverse=True)