1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Copyright 2014 Alexis Laferrière <alexis.laf@xymus.net>
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # Downloads files from RSS feeds
24 # Lazy man's verbose option
25 var verbose
: Bool = args
.has
("-v") or args
.has
("--verbose") is lazy
28 # Program configuration
31 # Folders used to infer regex (may be empty)
32 var regex_source_folders
: Array[Path]
34 # Custom patterns (may be empty)
35 var custom_patterns
: Array[Pattern]
37 # Download destination
38 var download_destination_folder
: Path
40 # RSS feeds (needs at least one)
41 var rss_source_urls
: Array[Text]
43 # Path to the log file
46 # Unique path of files names to prevent double download (may be empty)
47 var unique_pattern
: Array[Pattern]
49 # Exception where we ignore uniqueness and can be downloaded again (may be empty)
50 var unique_exceptions
: Array[Pattern]
52 # XML tag used for pattern recognition
53 fun tag_title
: String do return "title"
55 # Action to apply on each selected RSS element
56 fun act_on
(element
: Element)
58 var local_path
= download_destination_folder
.to_s
/ element
.title
59 element
.download_to
(local_path
)
63 # An element from an RSS feed
68 # Link to the file to download
71 redef fun to_s
do return "{title} @ {link}"
73 # Download this element to `path`
74 fun download_to
(path
: Text)
76 var request
= new CurlHTTPRequest(link
)
77 var response
= request
.download_to_file
(path
.to_s
)
79 if response
isa CurlResponseFailed then
80 sys
.stderr
.write
"Failed downloading URL '{link}' with: {response.error_msg} ({response.error_code})\n"
84 # Get an unique identifier for this element, uses `Config::unique_pattern`
85 fun unique_id
(config
: Config): String
87 for re
in config
.unique_pattern
do
88 var match
= title
.search
(re
)
90 return title
.substring
(0, match
.after
).to_lower
97 # Is this element except from uniqueness?
98 fun is_unique_exception
(config
: Config): Bool
100 for re
in config
.unique_exceptions
do
101 if title
.has
(re
) then
109 # Main program structure
114 # Local history (read from, them written to file)
115 var history
= new HashSet[Text]
120 # Read old log from file
121 if config
.log_path
.exists
then
122 var stream
= config
.log_path
.open_ro
123 history
.add_all stream
.read_all
.split
("\n")
127 # Get the pattern to search for
128 var patterns
= self.patterns
130 # Get all the elements from the RSS feeds
131 var elements
= new HashSet[Element]
132 for rss_url
in config
.rss_source_urls
do
133 var rss
= rss_url
.fetch_rss_content
134 elements
.add_all rss
.to_rss_elements
137 # Select the elements matching our pattern
138 var matches
= new HashSet[Element]
139 for pattern
in patterns
do for element
in elements
do
140 if element
.title
.has
(pattern
) then
146 print
"\n# {matches.length} matching elements:"
147 print matches
.join
("\n")
148 print
"\n# Downloading..."
151 for element
in matches
do
152 var unique_id
= element
.unique_id
(config
)
154 if history
.has
(unique_id
) then
155 # Do not download a file that is not unique according to `unique_id`
156 if not element
.is_unique_exception
(config
) then
157 # We make some exceptions
158 if sys
.verbose
then print
"File in log, skipping {element}"
164 if sys
.verbose
then print
"Acting on {element}"
166 tool_config
.act_on element
168 # Add `unique_id` to log
169 history
.add unique_id
172 # Save new log to file
173 var stream
= config
.log_path
.open_wo
174 for line
in history
do
181 # Gather all patterns from `Config::custom_patterns` and `Config::source_folder_path`
182 fun patterns
: Array[Pattern]
184 var patterns
= new Array[Pattern]
186 # Begin with custom pattern
187 for pattern_source
in config
.custom_patterns
do
188 patterns
.add pattern_source
191 # Get regex source from folder names
192 var folder_names
= new HashSet[Text]
193 for source_folder_path
in config
.regex_source_folders
do
194 var source_folder
= source_folder_path
196 if not source_folder
.exists
then
197 sys
.stderr
.write
"Regex source folder '{source_folder_path}' does not exists.\n"
201 for dir
in source_folder
.files
do if dir
.stat
.is_dir
then
202 folder_names
.add dir
.filename
206 # Compile our infered patterns
207 for folder_name
in folder_names
do
208 # Transform from "Some folder name" to "^Some.folder.name"
209 var regex_source
= folder_name
.
210 replace
(' ', ".").replace
('[', "\\[").replace
('(', "\\(").
211 replace
('+', "\\+").replace
('*', "\\*")
212 regex_source
= "^" + regex_source
214 var regex
= regex_source
.to_re
215 regex
.ignore_case
= true
220 if patterns
.is_empty
then
221 sys
.stderr
.write
"Do not have any pattern to work with.\n"
226 print
"# Generated {patterns.length} patterns"
227 print patterns
.join
("\n")
235 # Get the content of the RSS feed at `self`
236 fun fetch_rss_content
: Text
238 if sys
.verbose
then print
"\n# Downloading RSS file from '{self}'"
240 var request
= new CurlHTTPRequest(to_s
)
241 var response
= request
.execute
243 if response
isa CurlResponseSuccess then
244 var body
= response
.body_str
245 if sys
.verbose
then print
"Download successful"
247 else if response
isa CurlResponseFailed then
248 sys
.stderr
.write
"Failed downloading URL '{self}' with: {response.error_msg} ({response.error_code})\n"
255 # Get this RSS feed content as an `Array[Element]`
256 fun to_rss_elements
: Array[Element]
259 var items
= xml
["rss"].first
["channel"].first
["item"]
261 var elements
= new Array[Element]
263 var title
= item
[tool_config
.tag_title
].first
.as(XMLStartTag).data
264 var link
= item
["link"].first
.as(XMLStartTag).data
266 elements
.add
new Element(title
, link
)
270 print
"# Found elements:"
271 print elements
.join
("\n")
278 # Implement this method in your module to configure this tool
279 fun tool_config
: nullable Config do return null
283 print
"This tool is not configured, take a look at the example `sample_config.nit`"
285 abort # For the flow only
288 var tool
= new Downloader(c
)