# Basic catalog generator for Nit packages
#
# See: <http://nitlanguage.org/catalog/>
#
# The tool scans packages and generates the HTML files of a catalog.
#
# ## Features
#
# * [X] scan packages and their `.ini`
# * [X] generate lists of packages
# * [X] generate a page per package with the readme and most metadata
# * [ ] link/include/be included in the documentation
# * [ ] propose `related packages`
# * [X] show directory content (a la nitls)
# * [X] gather git information from the working directory
# * [ ] gather git information from the repository
# * [ ] gather package information from github
# * [ ] gather people information from github
# * [X] reify people
# * [X] separate information gathering from rendering
# * [ ] move up information gathering in (existing or new) service modules
# * [X] add command line options
# * [ ] harden HTML (escaping, path injection, etc)
# * [ ] nitcorn server with RESTful API
#
# ## Issues and limitations
#
# The tool works likee the other tools and expects to find valid Nit source code in the directories
#
# * cruft and temporary files will be collected
# * missing source file (e.g. not yet generated by nitcc) will make information
#   incomplete (e.g. invalid module thus partial dependency and metrics)
#
# How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear.
module catalog

import md5 # To get gravatar images
import counter # For statistics
import modelize # To process and count classes and methods

redef class MPackage

	# Metadata related to this package
	var metadata = new MPackageMetadata(self)
end

# The metadata extracted from a MPackage
class MPackageMetadata

	# The mpacakge this metadata belongs to
	var mpackage: MPackage

	# Return the associated metadata from the `ini`, if any
	fun metadata(key: String): nullable String do
		var ini = mpackage.ini
		if ini == null then return null
		return ini[key]
	end

	# The consolidated list of tags
	var tags: Array[String] is lazy do
		var tags = new Array[String]
		var string = metadata("package.tags")
		if string == null then return tags
		for tag in string.split(",") do
			tag = tag.trim
			if tag.is_empty then continue
			tags.add tag
		end
		if tryit != null then tags.add "tryit"
		if apk != null then tags.add "apk"
		if tags.is_empty then tags.add "none"
		return tags
	end

	# The list of all maintainers
	var maintainers = new Array[Person]

	# The list of contributors
	var contributors = new Array[Person]

	# The date of the most recent commit
	var last_date: nullable String = null

	# The date of the oldest commit
	var first_date: nullable String = null

	# Key: package.maintainer`
	var maintainer: nullable String is lazy do return metadata("package.maintainer")

	# Key: `package.more_contributors`
	var more_contributors: Array[String] is lazy do
		var res = new Array[String]
		var string = metadata("package.more_contributors")
		if string == null then return res
		for c in string.split(",") do
			c = c.trim
			if c.is_empty then continue
			res.add c
		end
		return res
	end

	# Key: `package.license`
	var license: nullable String is lazy do return metadata("package.license")

	# Key: `upstream.tryit`
	var tryit: nullable String is lazy do return metadata("upstream.tryit")

	# Key: `upstream.apk`
	var apk: nullable String is lazy do return metadata("upstream.apk")

	# Key: `upstream.homepage`
	var homepage: nullable String is lazy do return metadata("upstream.homepage")

	# Key: `upstream.browse`
	var browse: nullable String is lazy do return metadata("upstream.browse")

	# Package git clone address
	var git: nullable String is lazy do return metadata("upstream.git")

	# Package issue tracker
	var issues: nullable String is lazy do return metadata("upstream.issues")
end

redef class Int
	# Returns `log(self+1)`. Used to compute score of packages
	fun score: Float do return (self+1).to_f.log
end

# A contributor/author/etc.
#
# It comes from git or the metadata
#
# TODO get more things from github by using the email as a key
# "https://api.github.com/search/users?q={email}+in:email"
class Person
	# The name. Eg "John Doe"
	var name: String is writable

	# The email, Eg "john.doe@example.com"
	var email: nullable String is writable

	# Some homepage. Eg "http://example.com/~jdoe"
	var page: nullable String is writable

	# Gravatar id
	var gravatar: nullable String is lazy do
		var email = self.email
		if email == null then return null
		return email.md5.to_lower
	end

	# The standard representation of a person.
	#
	# ~~~
	# var jd = new Person("John Doe", "john.doe@example.com", "http://example.com/~jdoe")
	# assert jd.to_s == "John Doe <john.doe@example.com> (http://example.com/~jdoe)"
	# ~~~
	#
	# It can be used as the input of `parse`.
	#
	# ~~~
	# var jd2 = new Person.parse(jd.to_s)
	# assert jd2.to_s == jd.to_s
	# ~~~
	redef fun to_s
	do
		var res = name
		var email = self.email
		if email != null then res += " <{email}>"
		var page = self.page
		if page != null then res += " ({page})"
		return res
	end

	# Crete a new person from its standard textual representation.
	#
	# ~~~
	# var jd = new Person.parse("John Doe <john.doe@example.com> (http://example.com/~jdoe)")
	# assert jd.name == "John Doe"
	# assert jd.email == "john.doe@example.com"
	# assert jd.page == "http://example.com/~jdoe"
	# ~~~
	#
	# Emails and page are optional.
	#
	# ~~~
	# var jd2 = new Person.parse("John Doe")
	# assert jd2.name == "John Doe"
	# assert jd2.email == null
	# assert jd2.page == null
	# ~~~
	init parse(person: String)
	do
		var name = person
		var email = null
		var page = null
		# Regular expressions are broken, need to investigate.
		# So split manually.
		#
		#var re = "([^<(]*?)(<([^>]*?)>)?(\\((.*)\\))?".to_re
		#var m = (person+" ").search(re)
		#print "{person}: `{m or else "?"}` `{m[1] or else "?"}` `{m[3] or else "?"}` `{m[5] or else "?"}`"
		do
			var sp1 = person.split_once_on("<")
			if sp1.length < 2 then
				break
			end
			var sp2 = sp1.last.split_once_on(">")
			if sp2.length < 2 then
				break
			end
			name = sp1.first.trim
			email = sp2.first.trim
			var sp3 = sp2.last.split_once_on("(")
			if sp3.length < 2 then
				break
			end
			var sp4 = sp3.last.split_once_on(")")
			if sp4.length < 2 then
				break
			end
			page = sp4.first.trim
		end

		init(name, email, page)
	end
end


# The main class of the calatog generator that has the knowledge
class Catalog

	# The modelbuilder
	# used to access the files and count source lines of code
	var modelbuilder: ModelBuilder

	# List of all packages by their names
	var mpackages = new HashMap[String, MPackage]

	# Packages by tag
	var tag2proj = new MultiHashMap[String, MPackage]

	# Packages by category
	var cat2proj = new MultiHashMap[String, MPackage]

	# Packages by maintainer
	var maint2proj = new MultiHashMap[Person, MPackage]

	# Packages by contributors
	var contrib2proj = new MultiHashMap[Person, MPackage]

	# Dependency between packages
	fun deps: HashDigraph[MPackage] do return modelbuilder.model.mpackage_importation_graph

	# Number of modules by package
	var mmodules = new Counter[MPackage]

	# Number of classes by package
	var mclasses = new Counter[MPackage]

	# Number of methods by package
	var mmethods = new Counter[MPackage]

	# Number of line of code by package
	var loc = new Counter[MPackage]

	# Number of errors
	var errors = new Counter[MPackage]

	# Number of warnings and advices
	var warnings = new Counter[MPackage]

	# Number of warnings per 1000 lines of code (w/kloc)
	var warnings_per_kloc = new Counter[MPackage]

	# Documentation score (between 0 and 100)
	var documentation_score = new Counter[MPackage]

	# Number of commits by package
	var commits = new Counter[MPackage]

	# Score by package
	#
	# The score is loosely computed using other metrics
	var score = new Counter[MPackage]

	# List of known people by their git string
	var persons = new HashMap[String, Person]

	# Map person short names to person objects
	var name2person = new HashMap[String, Person]

	# Package statistics cache
	var mpackages_stats = new HashMap[MPackage, MPackageStats]

	# Scan, register and add a contributor to a package
	fun register_contrib(person: String, mpackage: MPackage): Person
	do
		var p = persons.get_or_null(person)
		if p == null then
			var new_p = new Person.parse(person)
			# Maybe, we already have this person in fact?
			p = persons.get_or_null(new_p.to_s)
			if p == null then
				p = new_p
				persons[p.to_s] = p
			end
		end
		var projs = contrib2proj[p]
		if not projs.has(mpackage) then
			projs.add mpackage
			mpackage.metadata.contributors.add p
		end
		name2person[p.name] = p
		return p
	end

	# Compute information for a package
	fun package_page(mpackage: MPackage)
	do
		mpackages[mpackage.full_name] = mpackage

		var score = score[mpackage].to_f

		var mdoc = mpackage.mdoc_or_fallback
		if mdoc != null then
			score += 100.0
			score += mdoc.content.length.score
		end
		var metadata = mpackage.metadata

		var tryit = metadata.tryit
		if tryit != null then
			score += 1.0
		end
		var apk = metadata.apk
		if apk != null then
			score += 1.0
		end
		var homepage = metadata.homepage
		if homepage != null then
			score += 5.0
		end
		var maintainer = metadata.maintainer
		if maintainer != null then
			score += 5.0
			var person = register_contrib(maintainer, mpackage)
			mpackage.metadata.maintainers.add person
			var projs = maint2proj[person]
			if not projs.has(mpackage) then projs.add mpackage
		end
		var license = metadata.license
		if license != null then
			score += 5.0
		end
		var browse = metadata.browse
		if browse != null then
			score += 5.0
		end
		var tags = metadata.tags
		for tag in tags do
			tag2proj[tag].add mpackage
		end
		if tags.not_empty then
			var cat = tags.first
			cat2proj[cat].add mpackage
			score += tags.length.score
		end
		if deps.has_vertex(mpackage) then
			score += deps.predecessors(mpackage).length.score
			score += deps.get_all_predecessors(mpackage).length.score
			score += deps.successors(mpackage).length.score
			score += deps.get_all_successors(mpackage).length.score
		end

		var contributors = mpackage.metadata.contributors
		var more_contributors = metadata.more_contributors
		for c in more_contributors do
			register_contrib(c, mpackage)
		end
		score += contributors.length.to_f
		var mmodules = 0
		var mclasses = 0
		var mmethods = 0
		var loc = 0
		var errors = 0
		var warnings = 0
		# The documentation value of each entity is ad hoc.
		var entity_score = 0.0
		var doc_score = 0.0
		for g in mpackage.mgroups do
			mmodules += g.mmodules.length
			var gs = 1.0
			entity_score += gs
			if g.mdoc != null then doc_score += gs
			for m in g.mmodules do
				var source = m.location.file
				if source != null then
					for msg in source.messages do
						if msg.level == 2 then
							errors += 1
						else
							warnings += 1
						end
					end
				end
				var am = modelbuilder.mmodule2node(m)
				if am != null then
					var file = am.location.file
					if file != null then
						loc += file.line_starts.length - 1
					end
				end
				var ms = gs
				if m.is_test then ms /= 100.0
				entity_score += ms
				if m.mdoc != null then doc_score += ms else ms /= 10.0
				for cd in m.mclassdefs do
					var cs = ms * 0.2
					if not cd.is_intro then cs /= 100.0
					if not cd.mclass.visibility <= private_visibility then cs /= 100.0
					entity_score += cs
					if cd.mdoc != null then doc_score += cs
					mclasses += 1
					for pd in cd.mpropdefs do
						var ps = ms * 0.1
						if not pd.is_intro then ps /= 100.0
						if not pd.mproperty.visibility <= private_visibility then ps /= 100.0
						entity_score += ps
						if pd.mdoc != null then doc_score += ps
						if not pd isa MMethodDef then continue
						mmethods += 1
					end
				end
			end
		end
		self.mmodules[mpackage] = mmodules
		self.mclasses[mpackage] = mclasses
		self.mmethods[mpackage] = mmethods
		self.loc[mpackage] = loc
		self.errors[mpackage] = errors
		self.warnings[mpackage] = warnings
		if loc > 0 then
			self.warnings_per_kloc[mpackage] = warnings * 1000 / loc
		end
		var documentation_score =  (100.0 * doc_score / entity_score).to_i
		self.documentation_score[mpackage] = documentation_score
		#score += mmodules.score
		score += mclasses.score
		score += mmethods.score
		score += loc.score
		score += documentation_score.score

		self.score[mpackage] = score.to_i
	end

	# Collect more information on a package using the `git` tool.
	fun git_info(mpackage: MPackage)
	do
		var ini = mpackage.ini
		if ini == null then return

		var root = mpackage.root
		if root == null then return

		# TODO use real git info
		#var repo = ini.get_or_null("upstream.git")
		#var branch = ini.get_or_null("upstream.git.branch")
		#var directory = ini.get_or_null("upstream.git.directory")

		var dirpath = root.filepath
		if dirpath == null then return

		# Collect commits info
		var res = git_run("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath)
		var contributors = new Counter[String]
		var commits = res.split("\n")
		if commits.not_empty and commits.last == "" then commits.pop
		self.commits[mpackage] = commits.length
		for l in commits do
			var s = l.split_once_on(';')
			if s.length != 2 or s.last == "" then continue

			# Collect date of last and first commit
			if mpackage.metadata.last_date == null then mpackage.metadata.last_date = s.first
			mpackage.metadata.first_date = s.first

			# Count contributors
			contributors.inc(s.last)
		end
		for c in contributors.sort.reverse_iterator do
			register_contrib(c, mpackage)
		end
	end

	# Compose package stats
	fun mpackage_stats(mpackage: MPackage): MPackageStats do
		var stats = new MPackageStats
		stats.mmodules = mmodules[mpackage]
		stats.mclasses = mclasses[mpackage]
		stats.mmethods = mmethods[mpackage]
		stats.loc = loc[mpackage]
		stats.errors = errors[mpackage]
		stats.warnings = warnings[mpackage]
		stats.warnings_per_kloc = warnings_per_kloc[mpackage]
		stats.documentation_score = documentation_score[mpackage]
		stats.commits = commits[mpackage]
		stats.score = score[mpackage]

		mpackages_stats[mpackage] = stats
		return stats
	end

	# Compose catalog stats
	var catalog_stats: CatalogStats is lazy do
		var stats = new CatalogStats
		stats.packages = mpackages.length
		stats.maintainers = maint2proj.length
		stats.contributors = contrib2proj.length
		stats.tags = tag2proj.length
		stats.modules = mmodules.sum
		stats.classes = mclasses.sum
		stats.methods = mmethods.sum
		stats.loc = loc.sum
		return stats
	end
end

# Catalog statistics
class CatalogStats

	# Number of packages
	var packages = 0

	# Number of maintainers
	var maintainers = 0

	# Number of contributors
	var contributors = 0

	# Number of tags
	var tags = 0

	# Number of modules
	var modules = 0

	# Number of classes
	var classes = 0

	# Number of methods
	var methods = 0

	# Number of line of codes
	var loc = 0

	# Return the stats as a Map associating each stat key to its value
	fun to_map: Map[String, Int] do
		var map = new HashMap[String, Int]
		map["packages"] = packages
		map["maintainers"] = maintainers
		map["contributors"] = contributors
		map["tags"] = tags
		map["modules"] = modules
		map["classes"] = classes
		map["methods"] = methods
		map["loc"] = loc
		return map
	end
end

# MPackage statistics for the catalog
class MPackageStats

	# Number of modules
	var mmodules = 0

	# Number of classes
	var mclasses = 0

	# Number of methods
	var mmethods = 0

	# Number of lines of code
	var loc = 0

	# Number of errors
	var errors = 0

	# Number of warnings and advices
	var warnings = 0

	# Number of warnings per 1000 lines of code (w/kloc)
	var warnings_per_kloc = 0

	# Documentation score (between 0 and 100)
	var documentation_score = 0

	# Number of commits by package
	var commits = 0

	# Score by package
	#
	# The score is loosely computed using other metrics
	var score = 0
end

# Sort the mpackages by their score
class CatalogScoreSorter
	super Comparator

	# Catalog used to access scores
	var catalog: Catalog

	redef type COMPARED: MPackage

	redef fun compare(a, b) do
		if not catalog.mpackages_stats.has_key(a) then return 1
		if not catalog.mpackages_stats.has_key(b) then return -1
		var astats = catalog.mpackages_stats[a]
		var bstats = catalog.mpackages_stats[b]
		return bstats.score <=> astats.score
	end
end

# Sort tabs alphabetically
class CatalogTagsSorter
	super Comparator

	redef type COMPARED: String

	redef fun compare(a, b) do return a <=> b
end

# Execute a git command and return the result
fun git_run(command: String...): String
do
	# print "git {command.join(" ")}"
	var p = new ProcessReader("git", command...)
	var res = p.read_all
	p.close
	p.wait
	return res
end