lib/popcorn/pop_tracker.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Popcorn web tracker
  16 #
  17 # Easy and ready to use web tracker you can apply to your popcorn application.
  18 #
  19 # The only thing you have to do is to use the tracker in your app routes:
  20
  21 # ~~~nitish
  22 # var config = new AppConfig
  23 # var app = new App
  24 # app.use("/", new HomeHandler)
  25 # app.use("/products", new ProductsHandler)
  26 # app.use("customers/", new CustomersHandler)
  27 #
  28 # app.use_after("/*", new PopTracker(config)) # tracker listens to /*
  29 # ~~~
  30 #
  31 # You can also use multiple tracker at once on different route.
  32 # All the data will be aggregated for you.
  33
  34 # ~~~nitish
  35 # app.use_after("/api/*", new PopTracker(config))
  36 # app.use_after("/admin/*", new PopTracker(config))
  37 # ~~~
  38 #
  39 # To retrieve your tracker data use the `PopTrackerAPI` which serves the tracker
  40 # data in JSON format.
  41 #
  42 # ~~~nitish
  43 # app.use("/api/tracker_data", new PopTrackerAPI(config))
  44 # ~~~
  45 module pop_tracker
  46
  47 import popcorn
  48 import popcorn::pop_config
  49 import popcorn::pop_logging
  50 import popcorn::pop_json
  51 import popcorn::pop_repos
  52
  53 redef class AppConfig
  54
  55         # Logs collection
  56         var tracker_logs = new TrackerRepo(db.collection("tracker_logs"))
  57
  58         # Tracker handler
  59         var tracker = new PopTracker(self)
  60 end
  61
  62 # JSON API of the PopTracker
  63 #
  64 # Serves the collected data in JSON format.
  65 class PopTrackerAPI
  66         super Router
  67
  68         # Config used to access tracker db
  69         var config: AppConfig
  70
  71         init do
  72                 use("/entries", new PopTrackerEntries(config))
  73                 use("/queries", new PopTrackerQueries(config))
  74                 use("/browsers", new PopTrackerBrowsers(config))
  75                 use("/times", new PopTrackerResponseTime(config))
  76         end
  77 end
  78
  79 # Base tracker handler
  80 abstract class TrackerHandler
  81         super Handler
  82
  83         # Config used to access tracker db
  84         var config: AppConfig
  85
  86         # Get the `limit` GET argument from `req`
  87         #
  88         # Return `10` by default.
  89         fun limit(req: HttpRequest): Int do return req.int_arg("limit") or else 10
  90 end
  91
  92 # Saves logs into a MongoDB collection
  93 class PopTracker
  94         super ConsoleLog
  95         super TrackerHandler
  96
  97         redef fun all(req, res) do
  98                 config.tracker_logs.save new LogEntry(req, res)
  99         end
 100 end
 101
 102 # List all tracker log entries
 103 class PopTrackerEntries
 104         super TrackerHandler
 105
 106         redef fun get(req, res) do
 107                 res.json new JsonArray.from(config.tracker_logs.find_all)
 108         end
 109 end
 110
 111 # Group and count entries by query string
 112 class PopTrackerQueries
 113         super TrackerHandler
 114
 115         redef fun get(req, res) do
 116                 var pipe = new MongoPipeline
 117                 pipe.group((new MongoGroup("$request.uri")).
 118                         sum("visits", 1).
 119                         avg("response_time", "$response_time").
 120                         addToSet("uniq", "$session"))
 121                 pipe.sort((new MongoMatch).eq("visits", -1))
 122                 pipe.limit(limit(req))
 123                 res.json new JsonArray.from(config.tracker_logs.collection.aggregate(pipe))
 124         end
 125 end
 126
 127 # Group and count entries by browser
 128 class PopTrackerBrowsers
 129         super TrackerHandler
 130
 131         # MongoMatch query used for each browser key
 132         #
 133         # Because parsing user agent string is a pain in the nit, we go lazy on this
 134         # one. We associate each broswer key like `Chromium` to the query that allows
 135         # us to count the number of visits.
 136         var browser_queries: HashMap[String, MongoMatch] do
 137                 var map = new HashMap[String, MongoMatch]
 138                 map["Chromium"] = (new MongoMatch).regex("user_agent", "Chromium")
 139                 map["Edge"] = (new MongoMatch).regex("user_agent", "Edge")
 140                 map["Firefox"] = (new MongoMatch).regex("user_agent", "Firefox")
 141                 map["IE"] = (new MongoMatch).regex("user_agent", "(MSIE)|(Trident)")
 142                 map["Netscape"] = (new MongoMatch).regex("user_agent", "Netscape")
 143                 map["Opera"] = (new MongoMatch).regex("user_agent", "Opera")
 144                 map["Safari"] = (new MongoMatch).land(null, [
 145                                 (new MongoMatch).regex("user_agent", "Safari"),
 146                                 (new MongoMatch).regex("user_agent", "^((?!Chrome).)*$")])
 147                 map["Chrome"] = (new MongoMatch).land(null, [
 148                                 (new MongoMatch).regex("user_agent", "Chrome"),
 149                                 (new MongoMatch).regex("user_agent", "^((?!Edge).)*$")])
 150
 151                 return map
 152         end
 153
 154         # Apply the `query` on `TrackerRepo::count`
 155         fun browser_count(query: MongoMatch): Int do return config.tracker_logs.count(query)
 156
 157         redef fun get(req, res) do
 158                 var browsers = new Array[BrowserCount]
 159                 for browser, query in self.browser_queries do
 160                         var count = new BrowserCount(browser, browser_count(query))
 161                         if count.count > 0 then browsers.add count
 162                 end
 163                 var sum = 0
 164                 for browser in browsers do sum += browser.count
 165                 var other = config.tracker_logs.count - sum
 166                 if other > 0 then browsers.add new BrowserCount("Other", other)
 167                 default_comparator.sort(browsers)
 168                 res.json new JsonArray.from(browsers)
 169         end
 170 end
 171
 172 # Associate each browser to its count.
 173 #
 174 # Only used to serialize the results.
 175 private class BrowserCount
 176         super Comparable
 177         super RepoObject
 178         serialize
 179
 180         redef type OTHER: SELF
 181
 182         var browser: String
 183         var count: Int
 184
 185         redef fun <=>(o) do return o.count <=> count
 186 end
 187
 188 # Return last month response time
 189 class PopTrackerResponseTime
 190         super TrackerHandler
 191
 192         redef fun get(req, res) do
 193                 var limit = get_time - (3600 * 24 * 30)
 194                 var pipe = new MongoPipeline
 195                 pipe.match((new MongoMatch).gte("timestamp", limit))
 196                 pipe.group((new MongoGroup("$timestamp")).
 197                         sum("visits", 1).
 198                         avg("response_time", "$response_time"))
 199                 pipe.sort((new MongoMatch).eq("_id", -1))
 200                 res.json new JsonArray.from(config.tracker_logs.collection.aggregate(pipe))
 201         end
 202 end
 203
 204 # A tracker log entry used to store HTTP requests and their given HTTP responses
 205 class LogEntry
 206         super RepoObject
 207         serialize
 208
 209         # HTTP request that triggered that log entry
 210         var request: HttpRequest
 211
 212         # HTTP response returned by the serveur
 213         var response: HttpResponse
 214
 215         # Request user-agent shortcut (easier for db requests
 216         var user_agent: nullable String is lazy do return request.header["User-Agent"]
 217
 218         # Processing time in miliseconds (or null if no clock was found in request)
 219         var response_time: nullable Int is lazy do
 220                 var clock = request.clock
 221                 if clock == null then return null
 222                 return (clock.total * 1000.0).to_i
 223         end
 224
 225         # Log entry timestamp
 226         var timestamp: Int = get_time
 227
 228         # Session ID associated to this entry
 229         var session: nullable String is lazy do
 230                 var session = request.session
 231                 if session == null then return null
 232                 return session.id_hash
 233         end
 234 end
 235
 236 # Repository to store track logs.
 237 class TrackerRepo
 238         super MongoRepository[LogEntry]
 239 end