#############################################################################################
# Generates plots and data files for various statistics processed on the raw data.
# Some of them concern the votes, other concern the rebellion index.
# 
# 07/2015 Israel Mendonça (v1)
# 10/2015 Vincent Labatut (v2)
#############################################################################################
source("src/define-constants.R")
source("src/plot-tools/plot-bars.R")
source("src/plot-tools/plot-histos.R")
source("src/prepare-data/filter-data.R")



#############################################################################################
# Processes basic statistics regarding the numbers of MEPs in each political group and for
# each member state.
#
# mep.details: informartion describing each MEP.
# plot.formats: formats of the plot files.
#############################################################################################
process.basic.stats <- function(mep.details, plot.formats)
{	tlog("..Processing MEPs distribution over groups and member states")
	
	# set folder up
	dir.create(MEPS.FOLDER, recursive=TRUE, showWarnings=FALSE)
	
	# MEPs by member state
	# absolute counts as bars
	title <- paste0("Number of MEPs by member state for the whole term")
	plot.file <- file.path(MEPS.FOLDER,"mep-counts-state")
	data <- plot.unif.indiv.raw.bars(plot.file, 
			bar.names=COUNTRY.VALUES, 
			values=mep.details[,COL.STATE],
			proportions=FALSE, areas=FALSE, y.lim=c(0,NA), 
			x.label="States", plot.title=title, 
			x.rotate=TRUE, format=plot.formats)
	# record as a table
	data <- data.frame(data)
	data <- cbind(data.frame(COUNTRY.VALUES),data)
	colnames(data) <- c(COL.STATE, COL.COUNT)
	table.file <- paste0(plot.file,".csv")
	write.csv2(data, file=table.file, row.names=FALSE)
	
	# proportions as bars
	title <- paste0("Proportion of MEPs by member state for the whole term")
	plot.file <- file.path(MEPS.FOLDER,"mep-proportions-state")
	data <- plot.unif.indiv.raw.bars(plot.file, 
			bar.names=COUNTRY.VALUES, 
			values=mep.details[,COL.STATE],
			proportions=TRUE, areas=FALSE, y.lim=c(0,1), 
			x.label="States", plot.title=title, 
			x.rotate=TRUE, format=plot.formats)
	# record as a table
	data <- data / sum(data)
	data[is.na(data)] <- 0		# remove possible /0 outcomes
	data <- data.frame(data)
	data <- cbind(data.frame(COUNTRY.VALUES),data)
	colnames(data) <- c(COL.STATE, COL.COUNT)
	table.file <- paste0(plot.file,".csv")
	write.csv2(data,file=table.file, row.names=FALSE)
	
	# MEPs by political group
	# absolute counts as bars
	title <- paste0("Number of MEPs by political group for the whole term")
	plot.file <- file.path(MEPS.FOLDER,"mep-counts-group")
	data <- plot.unif.indiv.raw.bars(plot.file, 
			bar.names=GROUP.VALUES, 
			values=mep.details[,COL.GROUP],
			proportions=FALSE, areas=FALSE, y.lim=c(0,NA), 
			x.label="Groups", plot.title=title, 
			x.rotate=FALSE, format=plot.formats)
	# record as a table
	data <- data.frame(data)
	data <- cbind(data.frame(GROUP.VALUES),data)
	colnames(data) <- c(COL.GROUP, COL.COUNT)
	table.file <- paste0(plot.file,".csv")
	write.csv2(data, file=table.file, row.names=FALSE)
	
	# proportions as bars
	title <- paste0("Proportion of MEPs by political group for the whole term")
	plot.file <- file.path(MEPS.FOLDER,"mep-proportions-group")
	data <- plot.unif.indiv.raw.bars(plot.file, 
			bar.names=GROUP.VALUES, 
			values=mep.details[,COL.GROUP],
			proportions=TRUE, areas=FALSE, y.lim=c(0,1), 
			x.label="Groups", plot.title=title, 
			x.rotate=FALSE, format=plot.formats)
	# record as a table
	data <- data / sum(data)
	data[is.na(data)] <- 0		# remove possible /0 outcomes
	data <- data.frame(data)
	data <- cbind(data.frame(GROUP.VALUES),data)
	colnames(data) <- c(COL.GROUP, COL.COUNT)
	table.file <- paste0(plot.file,".csv")
	write.csv2(data,file=table.file, row.names=FALSE)
}


#############################################################################################
# Counts the occurrences of policy domains among the voted documents. Generates the corresponding
# plots: distribution for the whole term and for each year.
#
# doc.details: table describing the voted documents.
# plot.formats: formats of the plot files.
#############################################################################################
process.domain.frequencies <- function(doc.details, plot.formats)
{	tlog("..Processing the frequencies of documents by policy domain")
	
	# set folder up
	dir.create(DOMAINS.FOLDER, recursive=TRUE, showWarnings=FALSE)
	
	# term domain distribution
	# absolute counts as bars
	title <- paste0("Numbers of documents by policy domain for the whole term")
	plot.file <- file.path(DOMAINS.FOLDER,"term-counts")
	data <- plot.unif.indiv.raw.bars(plot.file, 
			bar.names=DOMAIN.VALUES, 
			values=doc.details[,COL.DOMID],
			proportions=FALSE, areas=FALSE, y.lim=c(0,NA), 
			x.label="Domains", plot.title=title, 
			x.rotate=FALSE, format=plot.formats)
	# record as a table
	data <- data.frame(data)
	data <- cbind(data.frame(DOMAIN.VALUES),data)
	colnames(data) <- c(COL.DOMID, COL.COUNT)
	table.file <- paste0(plot.file,".csv")
	write.csv2(data,file=table.file, row.names=FALSE)
	
	# proportions as bars
	title <- paste0("Proportions of documents by policy domain - period=term")
	plot.file <- file.path(DOMAINS.FOLDER,"term-proportions")
	data <- plot.unif.indiv.raw.bars(plot.file, 
			bar.names=DOMAIN.VALUES, 
			values=doc.details[,COL.DOMID],
			proportions=TRUE, areas=FALSE, y.lim=c(0,1), 
			x.label="Domains", plot.title=title, 
			x.rotate=FALSE, format=plot.formats)
	# record as a table
	data <- data / sum(data)
	data[is.na(data)] <- 0		# remove possible /0 outcomes
	data <- data.frame(data)
	data <- cbind(data.frame(DOMAIN.VALUES),data)
	colnames(data) <- c(COL.DOMID, COL.COUNT)
	table.file <- paste0(plot.file,".csv")
	write.csv2(data,file=table.file, row.names=FALSE)
	
	# process the list of documents by year
	docs <- list()
	for(date in DATE.T7.YEARS)
	{	tlog("....Processing period ",DATE.STR.T7[date])
		
		# retain only the documents related to the selected dates
		docids <- filter.docs.by.date(doc.details, 
				start.date=DATE.START.T7[[date]], end.date=DATE.END.T7[[date]])
		idx <- match(docids,doc.details[,COL.DOCID])
		if(length(idx)>0)
			docs[[date]] <- doc.details[idx,COL.DOMID]
		else
			docs[[date]] <- NA
	}
	
	# yearly domain distribution
	# absolute counts as bars
	title <- paste0("Numbers of documents by policy domain - period=yearly")
	plot.file <- file.path(DOMAINS.FOLDER,"yearly-counts")
	data <- plot.stacked.indiv.raw.bars(plot.file, 
			bar.names=DATE.STR.T7[DATE.T7.YEARS], color.names=DOMAIN.VALUES, 
			values=docs, 
			proportions=FALSE, areas=FALSE, y.lim=c(0,NA), 
			x.label="Years", colors.label="Domains", plot.title=title, 
			x.rotate=FALSE, format=plot.formats)
	# record as a table
	data <- t(data.frame(data))
	data <- cbind(data.frame(DATE.STR.T7[DATE.T7.YEARS]),data)
	colnames(data) <- c(COL.DATE, DOMAIN.VALUES)
	table.file <- paste0(plot.file,".csv")
	write.csv2(data,file=table.file, row.names=FALSE)
	
	# proportions as bars
	title <- paste0("Proportions of documents by policy domain - period=yearly")
	plot.file <- file.path(DOMAINS.FOLDER,"yearly-proportions")
	data <- plot.stacked.indiv.raw.bars(plot.file, 
			bar.names=DATE.STR.T7[DATE.T7.YEARS], color.names=DOMAIN.VALUES, 
			values=docs, 
			proportions=TRUE, areas=FALSE, y.lim=c(0,1), 
			x.label="Years", colors.label="Domains", plot.title=title, 
			x.rotate=FALSE, format=plot.formats)
	# record as a table
	data <- lapply(data, function(v) if(all(v==0)) {return(rep(0,length(v)))} else {return(v/sum(v))})
	data <- t(data.frame(data))
	data <- cbind(data.frame(DATE.STR.T7[DATE.T7.YEARS]),data)
	colnames(data) <- c(COL.DATE, DOMAIN.VALUES)
	table.file <- paste0(plot.file,".csv")
	write.csv2(data,file=table.file, row.names=FALSE)
}


#############################################################################################
# Generate all plots and tables for the complete vote value distributions (FOR, AGAINST, etc.).
# The distributions of vote values are represented as bars vs. areas, for the absolute counts 
# vs. proportions, for each document.
#
# all.votes: raw vote data, including how each MEP voted.
# doc.details: description of each voted document.
# mep.details: description of each MEP.
# vote.values: the vote values we want to include in the plots and tables.
# file.prefix: string to use in the file names.
# colors.label: title of the colors legend.
# object: counted object (votes, loyal votes, etc.).
# domains: political domains to consider when processing the data.
# dates: time periods to consider when processing the data.
# country: member state currently processed (or NA if none in particular).
# group: political gorup currently processed (or NA if none in particular).
# plot.formats: formats of the plot files.
# vote.mode: whether the processed data are votes ("Vote"), behavior ("Behavior") or 
#			 participation ("Turnout").
#############################################################################################
process.stat.distribution.complete <- function(all.votes, doc.details, mep.details, vote.values, file.prefix, colors.label, object, domains, dates, country, group, plot.formats, vote.mode)
{	# setup file prefix
	if(is.na(file.prefix))
		file.prefix <- ""
	else
		file.prefix <- paste0(file.prefix,"-")
	# setup title prefix
	if(is.na(country))
		if(is.na(group))
			plot.prefix <- ""
		else
			plot.prefix <- paste0("[",group,"] ")
	else
		plot.prefix <- paste0("[",country,"] ")
	
	# consider each time period (each individual year as well as the whole term)
	for(date in dates)
	{	# consider each domain individually (including all domains at once)
#		for(dom in domains)
		foreach(dom=domains) %dopar% 
		{	source("src/define-imports.R")
			tlog("........Processing ",file.prefix," data for domain ",dom," and period ",DATE.STR.T7[date])
			
			# retain only the documents related to the selected topic and dates
			if(dom==DOMAIN.ALL)
				domval <- NA
			else
				domval <- dom
			filtered.doc.ids <- filter.docs.by.date.and.domain(doc.details, 
					start.date=DATE.START.T7[[date]], end.date=DATE.END.T7[[date]], 
					domains=domval)
			idx <- match(filtered.doc.ids,doc.details[,COL.DOCID])
			if(length(idx)<=1)
				tlog("........WARNING: Only ",length(idx)," documents remaining after filtering >> not processing these data")
			else
			{	# order the votes by date
				filtered.doc.details <- doc.details[idx,]
				indices <- order(as.Date(filtered.doc.details[,COL.DATE],"%d/%m/%Y"))
				
				# get the corresponding votes from the complete table
				cols <- match(filtered.doc.ids, colnames(all.votes))
				cols <- cols[indices]
				# remove the MEP who were inactive for all the considered votes
				active.idx <- which(apply(all.votes[,cols],1,function(v) !all(is.na(v))))
				# convert to list of column-vectors
				if(length(active.idx)==0)
					tlog("........WARNING: All votes are NAs (this can be correct, not necessarily a problem) >> not processing these data")
				else
				{	votes.temp <- all.votes[active.idx,cols,drop=FALSE]
					votes <- split(votes.temp, rep(1:ncol(votes.temp), each=nrow(votes.temp)))
					
					# setup folder
					#folder <- paste0(main.folder,dom,"/",DATE.STR.T7[date],"/")
					folder <- get.votes.path(vote=vote.mode, country, group, domain=dom, period=date)
					dir.create(folder, recursive=TRUE, showWarnings=FALSE)
					
					# absolute counts as bars
					#print(votes)				
					title <- paste0(plot.prefix,"Numbers of ",object," - domain=",dom," - period=",DATE.STR.T7[date])
					plot.file <- file.path(folder,paste0(file.prefix,"counts-bars"))
					data <- plot.stacked.indiv.raw.bars(plot.file, 
							bar.names=as.character(filtered.doc.ids[indices]), color.names=vote.values, 
							values=votes, 
							proportions=FALSE, areas=FALSE, y.lim=c(0,length(active.idx)), 
							x.label="Documents (sorted by date)", colors.label, plot.title=title, 
							x.rotate=TRUE, format=plot.formats)
					# absolute counts as areas
					plot.file <- file.path(folder,paste0(file.prefix,"counts-areas"))
					data <- plot.stacked.indiv.raw.bars(plot.file, 
							bar.names=1:length(filtered.doc.ids), color.names=vote.values, 
							values=votes, 
							proportions=FALSE, areas=TRUE, y.lim=c(0,length(active.idx)), 
							x.label="Documents (sorted by date)", colors.label, plot.title=title, 
							x.rotate=FALSE, format=plot.formats)
					# record as a table
					data <- t(data.frame(data))
					data <- cbind(data.frame(filtered.doc.ids[indices]),data)
					colnames(data) <- c(COL.DOCID, vote.values)
					table.file <- file.path(folder,paste0(file.prefix,"counts",".csv"))
					write.csv2(data,file=table.file, row.names=FALSE)
					
					# proportions as bars
					title <- paste0(plot.prefix,"Proportions of ",object," - domain=",dom," - period=",DATE.STR.T7[date])
					plot.file <- file.path(folder,paste0(file.prefix,"proportions-bars"))
					#print(plot.file);print(as.character(filtered.doc.ids[indices]));print(votes)
					data <- plot.stacked.indiv.raw.bars(plot.file, 
							bar.names=as.character(filtered.doc.ids[indices]), color.names=vote.values, 
							values=votes, 
							proportions=TRUE, areas=FALSE, y.lim=c(0,1), 
							x.label="Documents (sorted by date)", colors.label, plot.title=title, 
							x.rotate=TRUE, format=plot.formats)
					# proportions as areas
					plot.file <- file.path(folder,paste0(file.prefix,"proportions-areas"))
					data <- plot.stacked.indiv.raw.bars(plot.file, 
							bar.names=1:length(filtered.doc.ids), color.names=vote.values, 
							values=votes, 
							proportions=TRUE, areas=TRUE, y.lim=c(0,NA), 
							x.label="Documents (sorted by date)", colors.label, plot.title=title, 
							x.rotate=FALSE, format=plot.formats)
					# record as a table
					data <- lapply(data, function(v) if(all(v==0)) {return(rep(0,length(v)))} else {return(v/sum(v))})
					data <- t(data.frame(data))
					data <- cbind(data.frame(filtered.doc.ids[indices]),data)
					colnames(data) <- c(COL.DOCID, vote.values)
					table.file <- file.path(folder,paste0(file.prefix,"proportions",".csv"))
					write.csv2(data,file=table.file, row.names=FALSE)
					
					# record the individual data (i.e. MEP-related) as another table
					if(is.na(group) | is.na(country))
					{	data <- t(apply(votes.temp,1,function(r) c(table(factor(r,vote.values),useNA="no"))))
						data <- cbind(mep.details[active.idx,COL.MEPID],data)
						colnames(data)[1] <- COL.MEPID
						table.file <- file.path(folder,paste0(file.prefix,"indiv",".csv"))
						write.csv2(data,file=table.file, row.names=FALSE)
					}
				}
			}
		}
	}
}


#############################################################################################
# Generate all plots and tables for the aggregated vote value distributions (FOR, AGAINST, etc.).
# Unlike the "complete" variant, this function integrates data on various dimensions (temporal
# or topical). For each domain (including all domains at once), we aggregate the votes by year and for
# the whole term. The results are represented as bar plots, for both absolute counts and
# proportions.
#
# all.votes: raw vote data, including how each MEP voted.
# doc.details: description of each voted document.
# mep.details: description of each MEP.
# vote.values: the vote values we want to include in the plots and tables.
# file.prefix: string to use in the file names.
# colors.label: title of the colors legend.
# object: counted object (votes, loyal votes, etc.).
# domains: political domains to consider when processing the data.
# dates: time periods to consider when processing the data.
# country: member state currently processed (or NA if none in particular).
# group: political gorup currently processed (or NA if none in particular).
# plot.formats: formats of the plot files.
# vote.mode: whether the processed data are votes ("Vote"), behavior ("Behavior") or 
#			 participation ("Turnout").
#############################################################################################
process.stat.distribution.aggregate <- function(all.votes, doc.details, mep.details, vote.values, file.prefix, colors.label, object, domains, dates, country, group, plot.formats, vote.mode)
{	# setup file prefix
	if(is.na(file.prefix))
		file.prefix <- ""
	else
		file.prefix <- paste0(file.prefix,"-")
	# setup title prefix
	if(is.na(country))
		if(is.na(group))
			plot.prefix <- ""
		else
			plot.prefix <- paste0("[",group,"] ")
	else
		plot.prefix <- paste0("[",country,"] ")
	
	# consider each domain individually (including all domains at once)
	do.yearly <- FALSE
#	for(dom in domains)
	foreach(dom=domains) %dopar% 
	{	source("src/define-imports.R")
		tlog("........Processing ",file.prefix," data for domain ",dom)
		
		# consider each time period (each individual year as well as the whole term)
		votes <- list()
		votes.spe <- list()
		for(date in dates)
		{	tlog("..........Processing period ",DATE.STR.T7[date])
			
			# retain only the documents related to the selected topic and dates
			if(dom==DOMAIN.ALL)
				domval <- NA
			else
				domval <- dom
			filtered.doc.ids <- filter.docs.by.date.and.domain(doc.details, 
					start.date=DATE.START.T7[[date]], end.date=DATE.END.T7[[date]], 
					domains=domval)
			if(length(filtered.doc.ids)>0)
			{	cols <- match(filtered.doc.ids, colnames(all.votes))
				if(length(cols)==1)
					active.idx <- which(!is.na(all.votes[,cols]))
				else
					active.idx <- which(apply(all.votes[,cols],1,function(v) !all(is.na(v))))
				if(date==DATE.T7.TERM)
				{	votes.spe <- c(all.votes[active.idx,cols])
				}
				else
				{	votes[[date]] <- c(all.votes[active.idx,cols])
					do.yearly <- TRUE
				}
			}
			else
			{	votes[[date]] <- NA
				tlog("..........WARNING: Only ",length(filtered.doc.ids)," documents remaining after filtering >> not processing these data")
			}
		}
		
		# term-wise
		if(length(votes.spe)>0)
		{	if(all(is.na(votes.spe)))
				tlog("..........WARNING: All votes are NAs (this can be correct, not necessarily a problem) >> not processing these data")
			else if(all(unlist(votes.spe[!is.na(unlist(votes.spe))])==0))
				tlog("..........WARNING: All votes are 0s (this can be correct, not necessarily a problem) >> not processing these data")
			else
			{	# setup folder
				#folder <- paste0(main.folder,dom,"/aggregated/")
				folder <- get.votes.path(vote=vote.mode, country, group, domain=dom, period=DATE.T7.TERM)
				dir.create(folder, recursive=TRUE, showWarnings=FALSE)
				
				# absolute counts as bars
				title <- paste0(plot.prefix,"Numbers of ",object," - domain=",dom," - aggregation=term")
				plot.file <- file.path(folder,paste0("aggregated-term-",file.prefix,"counts"))
				data <- plot.unif.indiv.raw.bars(plot.file, 
						bar.names=vote.values, 
						values=votes.spe, 
						proportions=FALSE, areas=FALSE, y.lim=c(0,NA), 
						x.label=colors.label, plot.title=title, 
						x.rotate=FALSE, format=plot.formats)
				# record as a table
				data <- data.frame(data)
				data <- cbind(data.frame(vote.values),data)
				colnames(data) <- c(COL.VOTE, COL.COUNT)
				table.file <- paste0(plot.file,".csv")
				write.csv2(data,file=table.file, row.names=FALSE)
				
				# proportions as bars
				title <- paste0(plot.prefix,"Proportions of ",object," - domain=",dom," - aggregation=term")
				plot.file <- file.path(folder,paste0("aggregated-term-",file.prefix,"proportions"))
				data <- plot.unif.indiv.raw.bars(plot.file, 
						bar.names=vote.values, 
						values=votes.spe, 
						proportions=TRUE, areas=FALSE, y.lim=c(0,1), 
						x.label=colors.label, plot.title=title, 
						x.rotate=FALSE, format=plot.formats)
				# record as a table
				data <- data / sum(data)
				data[is.na(data)] <- 0
				data <- data.frame(data)
				data <- cbind(data.frame(vote.values),data)
				colnames(data) <- c(COL.VOTE, COL.PERCENT)
				table.file <- paste0(plot.file,".csv")
				write.csv2(data,file=table.file, row.names=FALSE)
			}
		}
		
		# by year
		if(do.yearly)
		{	if(all(is.na(votes)))
				tlog("..........WARNING: All yearly values are NAs (this can be correct, not necessarily a problem) >> not processing these data")
			else if(all(unlist(votes)[!is.na(unlist(votes))]==0))
				tlog("..........WARNING: All yearly values are 0s (this can be correct, not necessarily a problem) >> not processing these data")
			else
			{	# setup folder
				#folder <- paste0(main.folder,dom,"/aggregated/")
				folder <- get.votes.path(vote=vote.mode, country, group, domain=dom, period=DATE.T7.TERM)
				dir.create(folder, recursive=TRUE, showWarnings=FALSE)
				
				# absolute counts as bars
				title <- paste0(plot.prefix,"Numbers of ",object," - domain=",dom," aggregation=yearly")
				plot.file <- file.path(folder,paste0("aggregated-yearly-",file.prefix,"counts"))
				data <- plot.stacked.indiv.raw.bars(plot.file, 
						bar.names=DATE.STR.T7[DATE.T7.YEARS], color.names=vote.values, 
						values=votes, 
						proportions=FALSE, areas=FALSE, y.lim=c(0,NA), 
						x.label="Years", colors.label, plot.title=title, 
						x.rotate=FALSE, format=plot.formats)
				# record as a table
				data <- t(data.frame(data))
				temp <- matrix(NA,ncol=ncol(data),nrow=length(DATE.T7.YEARS))
				rownames(temp) <- DATE.T7.YEARS
				temp[rownames(data),] <- data
				data <- cbind(data.frame(DATE.STR.T7[DATE.T7.YEARS]),temp)
				colnames(data) <- c(COL.DATE, vote.values)
				table.file <- paste0(plot.file,".csv")
				write.csv2(data,file=table.file, row.names=FALSE)
				
				# proportions as bars
				title <- paste0(plot.prefix,"Proportions of ",object," - domain=",dom," aggregation=yearly")
				plot.file <- file.path(folder,paste0("aggregated-yearly-",file.prefix,"proportions"))
				data <- plot.stacked.indiv.raw.bars(plot.file, 
						bar.names=DATE.STR.T7[DATE.T7.YEARS], color.names=vote.values, 
						values=votes, 
						proportions=TRUE, areas=FALSE, y.lim=c(0,1), 
						x.label="Years", colors.label, plot.title=title, 
						x.rotate=FALSE, format=plot.formats)
				# record as a table
				data <- lapply(data, function(v) if(all(v==0)) {return(rep(0,length(v)))} else {return(v/sum(v))})
				data <- t(data.frame(data))
				temp <- matrix(NA,ncol=ncol(data),nrow=length(DATE.T7.YEARS))
				rownames(temp) <- DATE.T7.YEARS
				temp[rownames(data),] <- data
				data <- cbind(data.frame(DATE.STR.T7[DATE.T7.YEARS]),temp)
				colnames(data) <- c(COL.DATE, vote.values)
				table.file <- paste0(plot.file,".csv")
				write.csv2(data,file=table.file, row.names=FALSE)
			}
		}
	}
}


#############################################################################################
# Generate all plots and tables for the averaged vote value distributions (FOR, AGAINST, etc.).
# Unlike the "complete" variant, this function integrates data on various dimensions (temporal
# or topical). For each domain (including all domains), we average the votes by year and for
# the whole term. The resulting distributions are represented as histograms, for both absolute 
# counts and proportions.
#
# all.votes: raw vote data, including how each MEP voted.
# doc.details: description of each voted document.
# mep.details: description of each MEP.
# target: the vote value we want to count (to get numerical values, since we want an average).
# file.prefix: string to use in the file names.
# object: counted object (votes, loyal votes, etc.).
# domains: political domains to consider when processing the data.
# dates: time periods to consider when processing the data.
# country: member state currently processed (or NA if none in particular).
# group: political gorup currently processed (or NA if none in particular).
# plot.formats: formats of the plot files.
# vote.mode: whether the processed data are votes ("Vote"), behavior ("Behavior") or 
#			 participation ("Turnout").
#############################################################################################
process.stat.distribution.average <- function(all.votes, doc.details, mep.details, target, file.prefix, object, domains, dates, country, group, plot.formats, vote.mode)
{	# setup file prefix
	if(is.na(file.prefix))
		file.prefix <- ""
	else
		file.prefix <- paste0(file.prefix,"-")
	# setup title prefix
	if(is.na(country))
		if(is.na(group))
			plot.prefix <- ""
		else
			plot.prefix <- paste0("[",group,"] ")
	else
		plot.prefix <- paste0("[",country,"] ")
	
	# consider each domain individually (including all domains at once)
#	for(dom in domains)
	foreach(dom=domains) %dopar% 
	{	source("src/define-imports.R")
		tlog("........Processing ",file.prefix," data for domain ",dom)
		
		# consider each time period (each individual year as well as the whole term)
		for(date in dates)
		{	tlog("..........Processing period ",DATE.STR.T7[date])
			
			# setup folder
			#folder <- paste0(main.folder,dom,"/averaged/")
			folder <- get.votes.path(vote=vote.mode, country, group, domain=dom, period=date)
			dir.create(folder, recursive=TRUE, showWarnings=FALSE)
			
			# retain only the documents related to the selected topic and dates
			if(dom==DOMAIN.ALL)
				domval <- NA
			else
				domval <- dom
			filtered.doc.ids <- filter.docs.by.date.and.domain(doc.details, 
					start.date=DATE.START.T7[[date]], end.date=DATE.END.T7[[date]], 
					domains=domval)
			# check if there's enough data remaining
			if(length(filtered.doc.ids)>1)
			{	# format data
				cols <- match(filtered.doc.ids, colnames(all.votes))
				active.idx <- which(apply(all.votes[,cols],1,function(v) !all(is.na(v))))
				if(length(active.idx)>0)
				{	temp <- all.votes[active.idx,cols]
					numerized <- matrix(0,nrow=nrow(temp), ncol=ncol(temp))
					numerized[temp==target] <- 1
					votes <- apply(numerized, 2, mean)
					
					#print(votes)				
					# plot absolute counts as bars
					title <- paste0(plot.prefix,"Distribution of ",object," - domain=",dom,", - period=",DATE.STR.T7[date])
					plot.file <- file.path(folder,paste0("averaged-",file.prefix,"counts"))
					data <- plot.histo(plot.file, values=votes,
							x.label=object, 
							proportions=FALSE, x.lim=c(0,1), y.max=NA, break.nbr=NA, 
							plot.title=title, format=plot.formats)
					# record as a table
					data <- data[,c("y","xmin","xmax")]
					table.file <- paste0(plot.file,".csv")
					write.csv2(data,file=table.file, row.names=FALSE)
					
					# plot proportions as bars
					title <- paste0(plot.prefix,"Distribution of ",object," - domain=",dom,", - period=",DATE.STR.T7[date])
					plot.file <- file.path(folder,paste0("averaged-",file.prefix,"proportions"))
					data <- plot.histo(plot.file, values=votes,
							x.label=object, 
							proportions=TRUE, x.lim=c(0,1), y.max=0.5, break.nbr=NA, 
							plot.title=title, format=plot.formats)
					# record as a table
					data <- data[,c("y","xmin","xmax")]
					table.file <- paste0(plot.file,".csv")
					write.csv2(data,file=table.file, row.names=FALSE)
				}
				else
					tlog("..........WARNING: Only ",length(active.idx)," active MEPs after filtering >> not processing these data")
			}
			else
				tlog("..........WARNING: Only ",length(filtered.doc.ids)," documents remaining after filtering >> not processing these data")
		}
	}
}

#############################################################################################
# Deals with the generation of stats and plots related to the distribution of vote values 
# (FOR, AGAINST, etc.)
#
# all.votes: raw vote data, including how each MEP voted.
# doc.details: description of each voted document.
# mep.details: description of each MEP.
# domains: political domains to consider when processing the data.
# dates: time periods to consider when processing the data.
# country: member state currently processed (or NA if none in particular).
# group: political gorup currently processed (or NA if none in particular).
# plot.formats: formats of the plot files.
#############################################################################################
process.vote.distribution <- function(all.votes, doc.details, mep.details, domains, dates, country, group, plot.formats)
{	# process a simplified version of the data
	all.votes.smpl <- all.votes
	all.votes.smpl[all.votes.smpl==VOTE.ABSENT] <- VOTE.OTHER
	all.votes.smpl[all.votes.smpl==VOTE.ABST] <- VOTE.OTHER
	all.votes.smpl[all.votes.smpl==VOTE.DOCABSENT] <- VOTE.OTHER
	all.votes.smpl[all.votes.smpl==VOTE.NONE] <- VOTE.OTHER
	
	# setup parameters
	object <- "votes"
	colors.label <- "Votes"
	
	# process complete vote distributions
	tlog("......Plotting complete vote value distributions (raw votes)")
	process.stat.distribution.complete(all.votes=all.votes, doc.details, mep.details, vote.values=VOTE.VALUES, file.prefix="detailed", colors.label, object, domains, dates, country, group, plot.formats, vote.mode="Vote")
	tlog("......Plotting complete vote value distributions (simplified votes)")
	process.stat.distribution.complete(all.votes=all.votes.smpl, doc.details, mep.details, vote.values=VOTE.VALUES.SMPL, file.prefix="simplified", colors.label, object, domains, dates, country, group, plot.formats, vote.mode="Vote")
	
	# process aggregated vote distributions
	tlog("......Plotting aggregated vote values distributions (raw votes)")
	process.stat.distribution.aggregate(all.votes=all.votes, doc.details, mep.details, vote.values=VOTE.VALUES, file.prefix="detailed", colors.label, object, domains, dates, country, group, plot.formats, vote.mode="Vote")
	tlog("......Plotting aggregated vote values distributions (simplified votes)")
	process.stat.distribution.aggregate(all.votes=all.votes.smpl, doc.details, mep.details, vote.values=VOTE.VALUES.SMPL, file.prefix="simplified", colors.label, object, domains, dates, country, group, plot.formats, vote.mode="Vote")
}


#############################################################################################
# Deals with the generation of stats and plots related to the distribution of vote values 
# (FOR, AGAINST, etc.)
#
# behavior.values: individual behavior data, i.e. how each MEP voted relatively to his political 
#				   group (loyal or rebel).
# doc.details: description of each voted document.
# mep.details: description of each MEP.
# domains: political domains to consider when processing the data.
# dates: time periods to consider when processing the data.
# country: member state currently processed (or NA if none in particular).
# group: political gorup currently processed (or NA if none in particular).
# plot.formats: formats of the plot files.
#############################################################################################
process.behavior.stats <- function(behavior.values, doc.details, mep.details, domains, dates, country, group, plot.formats)
{	# process complete behavior distributions
	tlog("......Plotting complete behavior distributions")
	process.stat.distribution.complete(all.votes=behavior.values, doc.details, mep.details, vote.values=BEHAVIOR.VALUES, file.prefix=NA, colors.label="Behavior", object="loyal votes", domains, dates, country, group, plot.formats, vote.mode="Behavior")
	
	# process aggregated behavior distributions
	tlog("......Plotting aggregated behavior distributions")
	process.stat.distribution.aggregate(all.votes=behavior.values, doc.details, mep.details, vote.values=BEHAVIOR.VALUES, file.prefix=NA, colors.label="Behavior", object="loyal votes", domains, dates, country, group, plot.formats, vote.mode="Behavior")
	
	# process average behavior distributions
	tlog("......Plotting average behavior distributions")
	process.stat.distribution.average(all.votes=behavior.values, doc.details, mep.details, target=BEHAVIOR.LOYAL, file.prefix=NA, object="loyalty index", domains, dates, country, group, plot.formats, vote.mode="Behavior")
	
	# prepare the rebellion data
	behav.values <- behavior.values
	behav.values[behav.values==BEHAVIOR.LOYAL] <- NA
	group.behav.values <- apply(behav.values, 2, function(r) 
			{	idx <- which(r==BEHAVIOR.REBEL)
				if(length(idx)>0)
					r[idx] <- mep.details[idx,COL.GROUP]
				return(r)
			})
	colnames(group.behav.values) <- colnames(behavior.values)
	country.behav.values <- apply(behav.values, 2, function(r) 
			{	idx <- which(r==BEHAVIOR.REBEL)
				if(length(idx)>0)
					r[idx] <- COUNTRY.SHORTNAMES[mep.details[idx,COL.STATE]]
				return(r)
			})
	colnames(country.behav.values) <- colnames(behavior.values)
	
	# process complete rebellion distributions
	if(all(is.na(group)))
	{	tlog("......Plotting complete rebellion distributions for groups")
		process.stat.distribution.complete(all.votes=group.behav.values,    doc.details, mep.details, vote.values=GROUP.VALUES,       file.prefix="group-rebellion",   colors.label="Groups", object="rebel votes", domains, dates, country, group, plot.formats, vote.mode="Behavior")
	}
	if(all(is.na(country)))
	{	tlog("......Plotting complete rebellion distributions for countries")
		process.stat.distribution.complete(all.votes=country.behav.values,  doc.details, mep.details, vote.values=COUNTRY.SHORTNAMES, file.prefix="country-rebellion", colors.label="States", object="rebel votes", domains, dates, country, group, plot.formats, vote.mode="Behavior")
	}
	
	# process aggregated rebellion distributions
	if(all(is.na(group)))
	{	tlog("......Plotting aggregated rebellion distributions for groups")
		process.stat.distribution.aggregate(all.votes=group.behav.values,   doc.details, mep.details, vote.values=GROUP.VALUES,       file.prefix="group-rebellion",   colors.label="Groups", object="rebel votes", domains, dates, country, group, plot.formats, vote.mode="Behavior")
	}
	if(all(is.na(country)))
	{	tlog("......Plotting aggregated rebellion distributions for countries")
		process.stat.distribution.aggregate(all.votes=country.behav.values, doc.details, mep.details, vote.values=COUNTRY.SHORTNAMES, file.prefix="country-rebellion", colors.label="States", object="rebel votes", domains, dates, country, group, plot.formats, vote.mode="Behavior")
	}
}


#############################################################################################
# Deals with the generation of stats and plots related to the participation of MEPs, more
# precisely vote turnout and abstention.
#
# all.votes: raw vote data, including how each MEP voted.
# doc.details: description of each voted document.
# mep.details: description of each MEP.
# domains: political domains to consider when processing the data.
# dates: time periods to consider when processing the data.
# country: member state currently processed (or NA if none in particular).
# group: political gorup currently processed (or NA if none in particular).
# plot.formats: formats of the plot files.
#############################################################################################
process.turnout <- function(all.votes, doc.details, mep.details, domains, dates, country, group, plot.formats)
{	# prepare the expression data
	part.votes <- all.votes							# this is used to assess the relevance of vote similarity links, so we focus on expressed (only for/against) votes vs. the rest
	part.votes[part.votes==VOTE.ABSENT] <- NA
	part.votes[part.votes==VOTE.ABST] <- VOTE.OTHER
	part.votes[part.votes==VOTE.AGST] <- VOTE.EXPRESSED
	part.votes[part.votes==VOTE.FOR] <- VOTE.EXPRESSED
	part.votes[part.votes==VOTE.NONE] <- VOTE.OTHER
	part.votes[part.votes==VOTE.DOCABSENT] <- VOTE.OTHER
	part.votes[part.votes==VOTE.OTHER] <- VOTE.OTHER
	# prepare the abstention data
	abst.votes <- all.votes							# the abstention is processed by considering only the expressed votes
	abst.votes[abst.votes==VOTE.ABSENT] <- NA
	abst.votes[abst.votes==VOTE.ABST] <- VOTE.ABST
	abst.votes[abst.votes==VOTE.AGST] <- VOTE.OTHER
	abst.votes[abst.votes==VOTE.FOR] <- VOTE.OTHER
	abst.votes[abst.votes==VOTE.NONE] <- VOTE.ABST	# we consider not voting (while being present) as a form of abstention
	abst.votes[abst.votes==VOTE.DOCABSENT] <- NA
	abst.votes[abst.votes==VOTE.OTHER] <- VOTE.OTHER
	
	# process complete turnout distributions
	tlog("......Plotting complete turnout distributions")
	process.stat.distribution.complete(all.votes=part.votes, doc.details, mep.details, vote.values=c(VOTE.EXPRESSED,VOTE.OTHER), file.prefix="expr", colors.label="Expression", object="expressed votes", domains, dates, country, group, plot.formats, vote.mode="Turnout")
	process.stat.distribution.complete(all.votes=abst.votes, doc.details, mep.details, vote.values=c(VOTE.ABST, VOTE.OTHER), file.prefix="abst", colors.label="Abstention", object="abstain votes", domains, dates, country, group, plot.formats, vote.mode="Turnout")
	
	# process aggregated turnout distributions
	tlog("......Plotting aggregated turnout distributions")
	process.stat.distribution.aggregate(all.votes=part.votes, doc.details, mep.details, vote.values=c(VOTE.EXPRESSED,VOTE.OTHER), file.prefix="expr", colors.label="Expression", object="expressed votes", domains, dates, country, group, plot.formats, vote.mode="Turnout")
	process.stat.distribution.aggregate(all.votes=abst.votes, doc.details, mep.details, vote.values=c(VOTE.ABST, VOTE.OTHER), file.prefix="abst", colors.label="Abstention", object="abstain votes", domains, dates, country, group, plot.formats, vote.mode="Turnout")
	
	# process average turnout distributions
	tlog("......Plotting average turnout distributions")
	process.stat.distribution.average(all.votes=part.votes, doc.details, mep.details, target=VOTE.EXPRESSED, file.prefix="expr", object="expression index", domains, dates, country, group, plot.formats, vote.mode="Turnout")
	process.stat.distribution.average(all.votes=abst.votes, doc.details, mep.details, target=VOTE.ABST, file.prefix="abst", object="abstention index", domains, dates, country, group, plot.formats, vote.mode="Turnout")
	
	# prepare the absence data
	part.votes <- all.votes
	part.votes[part.votes==VOTE.ABSENT] <- VOTE.ABSENT
	part.votes[part.votes==VOTE.ABST] <- NA
	part.votes[part.votes==VOTE.AGST] <- NA
	part.votes[part.votes==VOTE.FOR] <- NA
	part.votes[part.votes==VOTE.NONE] <- NA
	part.votes[part.votes==VOTE.DOCABSENT] <- VOTE.ABSENT
	part.votes[part.votes==VOTE.OTHER] <- NA
	part.group.votes <- apply(part.votes, 2, function(r) 
			{	idx <- which(r==VOTE.ABSENT)
				if(length(idx)>0)
					r[idx] <- mep.details[idx,COL.GROUP]
				return(r)
			})
	colnames(part.group.votes) <- colnames(all.votes)
	part.country.votes <- apply(part.votes, 2, function(r) 
			{	idx <- which(r==VOTE.ABSENT)
				if(length(idx)>0)
					r[idx] <- COUNTRY.SHORTNAMES[mep.details[idx,COL.STATE]]
				return(r)
			})
	colnames(part.country.votes) <- colnames(all.votes)
	
	# prepare the abstention data
	abst.votes <- all.votes
	abst.votes[abst.votes==VOTE.ABSENT] <- NA
	abst.votes[abst.votes==VOTE.ABST] <- VOTE.ABST
	abst.votes[abst.votes==VOTE.AGST] <- NA
	abst.votes[abst.votes==VOTE.FOR] <- NA
	abst.votes[abst.votes==VOTE.NONE] <- NA
	abst.votes[abst.votes==VOTE.DOCABSENT] <- NA
	abst.votes[abst.votes==VOTE.OTHER] <- NA
	abst.group.votes <- apply(abst.votes, 2, function(r) 
			{	idx <- which(r==VOTE.ABST)
				if(length(idx)>0)
					r[idx] <- mep.details[idx,COL.GROUP]
				return(r)
			})
	colnames(abst.group.votes) <- colnames(all.votes)
	abst.country.votes <- apply(abst.votes, 2, function(r) 
			{	idx <- which(r==VOTE.ABST)
				if(length(idx)>0)
					r[idx] <- COUNTRY.SHORTNAMES[mep.details[idx,COL.STATE]]
				return(r)
			})
	colnames(abst.country.votes) <- colnames(all.votes)
	
	# process complete turnout distributions
	if(all(is.na(group)))
	{	tlog("......Plotting complete turnout distributions for groups")
		process.stat.distribution.complete(all.votes=part.group.votes, doc.details, mep.details, vote.values=GROUP.VALUES, file.prefix="group-abs", colors.label="Groups", object="absences", domains, dates, country, group, plot.formats, vote.mode="Turnout")
		process.stat.distribution.complete(all.votes=abst.group.votes, doc.details, mep.details, vote.values=GROUP.VALUES, file.prefix="group-abst",    colors.label="Groups", object="abstentions", domains, dates, country, group, plot.formats, vote.mode="Turnout")
	}
	if(all(is.na(country)))
	{	tlog("......Plotting complete turnout distributions for countries")
		process.stat.distribution.complete(all.votes=part.country.votes, doc.details, mep.details, vote.values=COUNTRY.SHORTNAMES, file.prefix="country-abs", colors.label="States", object="absences", domains, dates, country, group, plot.formats, vote.mode="Turnout")
		process.stat.distribution.complete(all.votes=abst.country.votes, doc.details, mep.details, vote.values=COUNTRY.SHORTNAMES, file.prefix="country-abst",    colors.label="States", object="abstentions", domains, dates, country, group, plot.formats, vote.mode="Turnout")
	}
	
	# process aggregated turnout distributions
	if(all(is.na(group)))
	{	tlog("......Plotting aggregated turnout distributions for groups")
		process.stat.distribution.aggregate(all.votes=part.group.votes, doc.details, mep.details, vote.values=GROUP.VALUES, file.prefix="group-abs", colors.label="Groups", object="absences", domains, dates, country, group, plot.formats, vote.mode="Turnout")
		process.stat.distribution.aggregate(all.votes=abst.group.votes, doc.details, mep.details, vote.values=GROUP.VALUES, file.prefix="group-abst",    colors.label="Groups", object="abstentions", domains, dates, country, group, plot.formats, vote.mode="Turnout")
	}
	if(all(is.na(country)))
	{	tlog("......Plotting aggregated turnout distributions for countries")
		process.stat.distribution.aggregate(all.votes=part.country.votes, doc.details, mep.details, vote.values=COUNTRY.SHORTNAMES, file.prefix="country-abs", colors.label="States", object="absences", domains, dates, country, group, plot.formats, vote.mode="Turnout")
		process.stat.distribution.aggregate(all.votes=abst.country.votes, doc.details, mep.details, vote.values=COUNTRY.SHORTNAMES, file.prefix="country-abst",    colors.label="States", object="abstentions", domains, dates, country, group, plot.formats, vote.mode="Turnout")
	}
}


#############################################################################################
# Main function of this script, generating all stat-related tables and plots.
#
# all.votes: individual vote data, i.e. how each MEP voted.
# behavior.values: individual behavior data, i.e. how each MEP voted relatively to his political 
#				   group (i.e. loyal or rebel).
# doc.details: description of each voted document.
# mep.details: description of each MEP.
# domains: political domains to consider when processing the data.
# dates: time periods to consider when processing the data.
# everything: whether to process all data without distinction of country or political group.
# countries: member states to consider separately when processing the data.
# groups: political groups to consider separately when processing the data.
# plot.formats: formats of the plot files.
#############################################################################################
process.stats <- function(all.votes, behavior.values, doc.details, mep.details, domains, dates, everything, countries, groups, plot.formats)
{	tlog("***************************************************")
	tlog("****** PROCESSING STATS")
	tlog("***************************************************")
	
	# domain stats
	process.domain.frequencies(doc.details, plot.formats)
	# group and country stats
	process.basic.stats(mep.details, plot.formats)
	
	# stats by political group
	tlog("..Process stats by group")
	for(group in groups)
	{	tlog("....Process stats for group ",group)
		
		# select data
		mepids <- filter.meps.by.group(mep.details,group)
		idx <- match(mepids,all.votes[,COL.MEPID])
		group.votes <- all.votes[idx,]
		group.behavior <- behavior.values[idx,]
		group.details <- mep.details[idx,]
		
		# process vote (for, against...) stats
		process.vote.distribution(group.votes, doc.details, group.details, domains, dates, country=NA, group, plot.formats)
		# process behavior (loyalty vs. rebel) stats
		process.behavior.stats(group.behavior, doc.details, group.details, domains, dates, country=NA, group, plot.formats)
		# process turnout (vote vs. absence) stats
		process.turnout(group.votes, doc.details, group.details, domains, dates, country=NA, group, plot.formats)
	}
	
	# stats by home country
	tlog("..Process stats by country")
	for(country in countries)
	#country <- COUNTRY.HR
	{	tlog("....Process stats for country ",country)
		
		# select data
		mepids <- filter.meps.by.country(mep.details,country)
		idx <- match(mepids,all.votes[,COL.MEPID])
		country.votes <- all.votes[idx,]
		country.behavior <- behavior.values[idx,]
		country.details <- mep.details[idx,]
		
		# process vote (for, against...) stats
		process.vote.distribution(country.votes, doc.details, country.details, domains, dates, country, group=NA, plot.formats)
		# process behavior (loyalty vs. rebel) stats
		process.behavior.stats(country.behavior, doc.details, country.details, domains, dates, country, group=NA, plot.formats)
		# process turnout (vote vs. absence) stats
		process.turnout(country.votes, doc.details, country.details, domains, dates, country, group=NA, plot.formats)
	}
	
	# process stats for all data
	if(everything)
	{	tlog("..Process stats for all data")
		process.vote.distribution(all.votes, doc.details, mep.details, domains, dates, country=NA, group=NA, plot.formats)
		process.behavior.stats(behavior.values, doc.details, mep.details, domains, dates, country=NA, group=NA, plot.formats)
		process.turnout(all.votes, doc.details, domains, dates, country=NA, group=NA, plot.formats)
	}
}
