lib/popcorn/pop_tracker.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Popcorn web tracker
  16 #
  17 # Easy and ready to use web tracker you can apply to your popcorn application.
  18 #
  19 # The only thing you have to do is to use the tracker in your app routes:
  20
  21 # ~~~nitish
  22 # var config = new AppConfig
  23 # var app = new App
  24 # app.use("/", new HomeHandler)
  25 # app.use("/products", new ProductsHandler)
  26 # app.use("customers/", new CustomersHandler)
  27 #
  28 # app.use_after("/*", new PopTracker(config)) # tracker listens to /*
  29 # ~~~
  30 #
  31 # You can also use multiple tracker at once on different route.
  32 # All the data will be aggregated for you.
  33
  34 # ~~~nitish
  35 # app.use_after("/api/*", new PopTracker(config))
  36 # app.use_after("/admin/*", new PopTracker(config))
  37 # ~~~
  38 #
  39 # To retrieve your tracker data use the `PopTrackerAPI` which serves the tracker
  40 # data in JSON format.
  41 #
  42 # ~~~nitish
  43 # app.use("/api/tracker_data", new PopTrackerAPI(config))
  44 # ~~~
  45 module pop_tracker
  46
  47 import popcorn
  48 import popcorn::pop_config
  49 import popcorn::pop_json
  50 import popcorn::pop_repos
  51
  52 redef class AppConfig
  53
  54         # Logs collection
  55         var tracker_logs = new TrackerRepo(db.collection("tracker_logs"))
  56
  57         # Tracker handler
  58         var tracker = new PopTracker(self)
  59 end
  60
  61 # JSON API of the PopTracker
  62 #
  63 # Serves the collected data in JSON format.
  64 class PopTrackerAPI
  65         super Router
  66
  67         # Config used to access tracker db
  68         var config: AppConfig
  69
  70         init do
  71                 use("/entries", new PopTrackerEntries(config))
  72                 use("/queries", new PopTrackerQueries(config))
  73                 use("/browsers", new PopTrackerBrowsers(config))
  74                 use("/times", new PopTrackerResponseTime(config))
  75         end
  76 end
  77
  78 # Base tracker handler
  79 abstract class TrackerHandler
  80         super Handler
  81
  82         # Config used to access tracker db
  83         var config: AppConfig
  84
  85         # Get the `limit` GET argument from `req`
  86         #
  87         # Return `10` by default.
  88         fun limit(req: HttpRequest): Int do return req.int_arg("limit") or else 10
  89 end
  90
  91 # Saves logs into a MongoDB collection
  92 class PopTracker
  93         super TrackerHandler
  94
  95         redef fun all(req, res) do
  96                 config.tracker_logs.save new LogEntry(req, res)
  97         end
  98 end
  99
 100 # List all tracker log entries
 101 class PopTrackerEntries
 102         super TrackerHandler
 103
 104         redef fun get(req, res) do
 105                 res.json new JsonArray.from(config.tracker_logs.find_all)
 106         end
 107 end
 108
 109 # Group and count entries by query string
 110 class PopTrackerQueries
 111         super TrackerHandler
 112
 113         redef fun get(req, res) do
 114                 var pipe = new MongoPipeline
 115                 pipe.group((new MongoGroup("$request.uri")).
 116                         sum("visits", 1).
 117                         avg("response_time", "$response_time").
 118                         addToSet("uniq", "$session"))
 119                 pipe.sort((new MongoMatch).eq("visits", -1))
 120                 pipe.limit(limit(req))
 121                 res.json new JsonArray.from(config.tracker_logs.collection.aggregate(pipe))
 122         end
 123 end
 124
 125 # Group and count entries by browser
 126 class PopTrackerBrowsers
 127         super TrackerHandler
 128
 129         # MongoMatch query used for each browser key
 130         #
 131         # Because parsing user agent string is a pain in the nit, we go lazy on this
 132         # one. We associate each broswer key like `Chromium` to the query that allows
 133         # us to count the number of visits.
 134         var browser_queries: HashMap[String, MongoMatch] do
 135                 var map = new HashMap[String, MongoMatch]
 136                 map["Chromium"] = (new MongoMatch).regex("user_agent", "Chromium")
 137                 map["Edge"] = (new MongoMatch).regex("user_agent", "Edge")
 138                 map["Firefox"] = (new MongoMatch).regex("user_agent", "Firefox")
 139                 map["IE"] = (new MongoMatch).regex("user_agent", "(MSIE)|(Trident)")
 140                 map["Netscape"] = (new MongoMatch).regex("user_agent", "Netscape")
 141                 map["Opera"] = (new MongoMatch).regex("user_agent", "Opera")
 142                 map["Safari"] = (new MongoMatch).land(null, [
 143                                 (new MongoMatch).regex("user_agent", "Safari"),
 144                                 (new MongoMatch).regex("user_agent", "^((?!Chrome).)*$")])
 145                 map["Chrome"] = (new MongoMatch).land(null, [
 146                                 (new MongoMatch).regex("user_agent", "Chrome"),
 147                                 (new MongoMatch).regex("user_agent", "^((?!Edge).)*$")])
 148
 149                 return map
 150         end
 151
 152         # Apply the `query` on `TrackerRepo::count`
 153         fun browser_count(query: MongoMatch): Int do return config.tracker_logs.count(query)
 154
 155         redef fun get(req, res) do
 156                 var browsers = new Array[BrowserCount]
 157                 for browser, query in self.browser_queries do
 158                         var count = new BrowserCount(browser, browser_count(query))
 159                         if count.count > 0 then browsers.add count
 160                 end
 161                 var sum = 0
 162                 for browser in browsers do sum += browser.count
 163                 var other = config.tracker_logs.count - sum
 164                 if other > 0 then browsers.add new BrowserCount("Other", other)
 165                 default_comparator.sort(browsers)
 166                 res.json new JsonArray.from(browsers)
 167         end
 168 end
 169
 170 # Associate each browser to its count.
 171 #
 172 # Only used to serialize the results.
 173 private class BrowserCount
 174         super Comparable
 175         super RepoObject
 176         serialize
 177
 178         redef type OTHER: SELF
 179
 180         var browser: String
 181         var count: Int
 182
 183         redef fun <=>(o) do return o.count <=> count
 184 end
 185
 186 # Return last month response time
 187 class PopTrackerResponseTime
 188         super TrackerHandler
 189
 190         redef fun get(req, res) do
 191                 var limit = get_time - (3600 * 24 * 30)
 192                 var pipe = new MongoPipeline
 193                 pipe.match((new MongoMatch).gte("timestamp", limit))
 194                 pipe.group((new MongoGroup("$timestamp")).
 195                         sum("visits", 1).
 196                         avg("response_time", "$response_time"))
 197                 pipe.sort((new MongoMatch).eq("_id", -1))
 198                 res.json new JsonArray.from(config.tracker_logs.collection.aggregate(pipe))
 199         end
 200 end
 201
 202 # A tracker log entry used to store HTTP requests and their given HTTP responses
 203 class LogEntry
 204         super RepoObject
 205         serialize
 206
 207         # HTTP request that triggered that log entry
 208         var request: HttpRequest
 209
 210         # HTTP response returned by the serveur
 211         var response: HttpResponse
 212
 213         # Request user-agent shortcut (easier for db requests
 214         var user_agent: nullable String is lazy do return request.header["User-Agent"]
 215
 216         # Processing time in miliseconds (or null if no clock was found in request)
 217         var response_time: nullable Int is lazy do
 218                 var clock = request.clock
 219                 if clock == null then return null
 220                 return (clock.total * 1000.0).to_i
 221         end
 222
 223         # Log entry timestamp
 224         var timestamp: Int = get_time
 225
 226         # Session ID associated to this entry
 227         var session: nullable String is lazy do
 228                 var session = request.session
 229                 if session == null then return null
 230                 return session.id_hash
 231         end
 232 end
 233
 234 # Repository to store track logs.
 235 class TrackerRepo
 236         super MongoRepository[LogEntry]
 237 end