lib/popcorn/pop_tracker.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Popcorn web tracker
  16 #
  17 # Easy and ready to use web tracker you can apply to your popcorn application.
  18 #
  19 # The only thing you have to do is to use the tracker in your app routes:
  20
  21 # ~~~nitish
  22 # var config = new AppConfig
  23 # var app = new App
  24 # app.use("/", new HomeHandler)
  25 # app.use("/products", new ProductsHandler)
  26 # app.use("customers/", new CustomersHandler)
  27 #
  28 # app.use_after("/*", new PopTracker(config)) # tracker listens to /*
  29 # ~~~
  30 #
  31 # You can also use multiple tracker at once on different route.
  32 # All the data will be aggregated for you.
  33
  34 # ~~~nitish
  35 # app.use_after("/api/*", new PopTracker(config))
  36 # app.use_after("/admin/*", new PopTracker(config))
  37 # ~~~
  38 #
  39 # To retrieve your tracker data use the `PopTrackerAPI` which serves the tracker
  40 # data in JSON format.
  41 #
  42 # ~~~nitish
  43 # app.use("/api/tracker_data", new PopTrackerAPI(config))
  44 # ~~~
  45 module pop_tracker
  46
  47 import popcorn
  48 import popcorn::pop_config
  49 import popcorn::pop_logging
  50 import popcorn::pop_repos
  51
  52 redef class AppConfig
  53
  54         # Logs collection
  55         var tracker_logs = new TrackerRepo(db.collection("tracker_logs"))
  56
  57         # Tracker handler
  58         var tracker = new PopTracker(self)
  59 end
  60
  61 # JSON API of the PopTracker
  62 #
  63 # Serves the collected data in JSON format.
  64 class PopTrackerAPI
  65         super Router
  66
  67         # Config used to access tracker db
  68         var config: AppConfig
  69
  70         init do
  71                 use("/entries", new PopTrackerEntries(config))
  72                 use("/queries", new PopTrackerQueries(config))
  73                 use("/browsers", new PopTrackerBrowsers(config))
  74                 use("/times", new PopTrackerResponseTime(config))
  75         end
  76 end
  77
  78 # Base tracker handler
  79 abstract class TrackerHandler
  80         super Handler
  81
  82         # Config used to access tracker db
  83         var config: AppConfig
  84
  85         # Get the `limit` GET argument from `req`
  86         #
  87         # Return `10` by default.
  88         fun limit(req: HttpRequest): Int do return req.int_arg("limit") or else 10
  89 end
  90
  91 # Saves logs into a MongoDB collection
  92 class PopTracker
  93         super ConsoleLog
  94         super TrackerHandler
  95
  96         redef fun all(req, res) do
  97                 config.tracker_logs.save new LogEntry(req, res)
  98         end
  99 end
 100
 101 # List all tracker log entries
 102 class PopTrackerEntries
 103         super TrackerHandler
 104
 105         redef fun get(req, res) do
 106                 res.json new JsonArray.from(config.tracker_logs.find_all)
 107         end
 108 end
 109
 110 # Group and count entries by query string
 111 class PopTrackerQueries
 112         super TrackerHandler
 113
 114         redef fun get(req, res) do
 115                 var pipe = new MongoPipeline
 116                 pipe.group((new MongoGroup("$request.uri")).
 117                         sum("visits", 1).
 118                         avg("response_time", "$response_time").
 119                         addToSet("uniq", "$session"))
 120                 pipe.sort((new MongoMatch).eq("visits", -1))
 121                 pipe.limit(limit(req))
 122                 res.json new JsonArray.from(config.tracker_logs.collection.aggregate(pipe))
 123         end
 124 end
 125
 126 # Group and count entries by browser
 127 class PopTrackerBrowsers
 128         super TrackerHandler
 129
 130         # MongoMatch query used for each browser key
 131         #
 132         # Because parsing user agent string is a pain in the nit, we go lazy on this
 133         # one. We associate each broswer key like `Chromium` to the query that allows
 134         # us to count the number of visits.
 135         var browser_queries: HashMap[String, MongoMatch] do
 136                 var map = new HashMap[String, MongoMatch]
 137                 map["Chromium"] = (new MongoMatch).regex("user_agent", "Chromium")
 138                 map["Edge"] = (new MongoMatch).regex("user_agent", "Edge")
 139                 map["Firefox"] = (new MongoMatch).regex("user_agent", "Firefox")
 140                 map["IE"] = (new MongoMatch).regex("user_agent", "(MSIE)|(Trident)")
 141                 map["Netscape"] = (new MongoMatch).regex("user_agent", "Netscape")
 142                 map["Opera"] = (new MongoMatch).regex("user_agent", "Opera")
 143                 map["Safari"] = (new MongoMatch).land(null, [
 144                                 (new MongoMatch).regex("user_agent", "Safari"),
 145                                 (new MongoMatch).regex("user_agent", "^((?!Chrome).)*$")])
 146                 map["Chrome"] = (new MongoMatch).land(null, [
 147                                 (new MongoMatch).regex("user_agent", "Chrome"),
 148                                 (new MongoMatch).regex("user_agent", "^((?!Edge).)*$")])
 149
 150                 return map
 151         end
 152
 153         # Apply the `query` on `TrackerRepo::count`
 154         fun browser_count(query: MongoMatch): Int do return config.tracker_logs.count(query)
 155
 156         redef fun get(req, res) do
 157                 var browsers = new Array[BrowserCount]
 158                 for browser, query in self.browser_queries do
 159                         var count = new BrowserCount(browser, browser_count(query))
 160                         if count.count > 0 then browsers.add count
 161                 end
 162                 var sum = 0
 163                 for browser in browsers do sum += browser.count
 164                 var other = config.tracker_logs.count - sum
 165                 if other > 0 then browsers.add new BrowserCount("Other", other)
 166                 default_comparator.sort(browsers)
 167                 res.json new JsonArray.from(browsers)
 168         end
 169 end
 170
 171 # Associate each browser to its count.
 172 #
 173 # Only used to serialize the results.
 174 private class BrowserCount
 175         super Comparable
 176         super RepoObject
 177         serialize
 178
 179         redef type OTHER: SELF
 180
 181         var browser: String
 182         var count: Int
 183
 184         redef fun <=>(o) do return o.count <=> count
 185 end
 186
 187 # Return last month response time
 188 class PopTrackerResponseTime
 189         super TrackerHandler
 190
 191         redef fun get(req, res) do
 192                 var limit = get_time - (3600 * 24 * 30)
 193                 var pipe = new MongoPipeline
 194                 pipe.match((new MongoMatch).gte("timestamp", limit))
 195                 pipe.group((new MongoGroup("$timestamp")).
 196                         sum("visits", 1).
 197                         avg("response_time", "$response_time"))
 198                 pipe.sort((new MongoMatch).eq("_id", -1))
 199                 res.json new JsonArray.from(config.tracker_logs.collection.aggregate(pipe))
 200         end
 201 end
 202
 203 # A tracker log entry used to store HTTP requests and their given HTTP responses
 204 class LogEntry
 205         super RepoObject
 206         serialize
 207
 208         # HTTP request that triggered that log entry
 209         var request: HttpRequest
 210
 211         # HTTP response returned by the serveur
 212         var response: HttpResponse
 213
 214         # Request user-agent shortcut (easier for db requests
 215         var user_agent: nullable String is lazy do return request.header["User-Agent"]
 216
 217         # Processing time in miliseconds (or null if no clock was found in request)
 218         var response_time: nullable Int is lazy do
 219                 var clock = request.clock
 220                 if clock == null then return null
 221                 return (clock.total * 1000.0).to_i
 222         end
 223
 224         # Log entry timestamp
 225         var timestamp: Int = get_time
 226
 227         # Session ID associated to this entry
 228         var session: nullable String is lazy do
 229                 var session = request.session
 230                 if session == null then return null
 231                 return session.id_hash
 232         end
 233 end
 234
 235 # Repository to store track logs.
 236 class TrackerRepo
 237         super MongoRepository[LogEntry]
 238 end