1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Crawler on the nitweb web API
20 # Download a HTTP resource
21 fun curl
(url
: String): String do
23 var p
= new ProcessReader("curl", "-s", url
)
27 # TODO: process HTTP error codes
29 print
"Error with {url}"
34 # Recursively collect all string values in a json value associated to a given key.
35 fun search_json
(json
: nullable Serializable, key
: String, result
: nullable Array[String]): Array[String]
37 if result
== null then result
= new Array[String]
38 if json
isa JsonObject then
40 search_json
(v
, key
, result
)
42 var v
= json
.get_or_null
(key
)
47 else if json
isa JsonArray then
48 for e
in json
do search_json
(e
, key
, result
)
49 else if json
== null or json
isa String or json
isa Int or json
isa Bool or json
isa Float then
58 var server
= "http://localhost:3000"
59 var todo
= ["/api/entity/core"]
61 if args
.not_empty
then
63 if args
.not_empty
then
69 var seen
= new Set[String]
73 while todo
.not_empty
do
77 print
"process {url}. {cpt}+{todo.length}/{seen.length}"
80 var txt
= js
.parse_json
82 print
"{url}: {txt.message}"
85 for x
in search_json
(txt
, "api_url") do
86 if seen
.has
(x
) then continue