1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Copyright 2014 Alexis Laferrière <alexis.laf@xymus.net>
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # Downloads files from RSS feeds
24 # Lazy man's verbose option
25 var verbose
: Bool = args
.has
("-v") or args
.has
("--verbose") is lazy
28 # Program configuration
31 # Folders used to infer regex (may be empty)
32 var regex_source_folders
: Array[Path]
34 # Custom patterns (may be empty)
35 var custom_patterns
: Array[Pattern]
37 # Download destination
38 var download_destination_folder
: Path
40 # RSS feeds (needs at least one)
41 var rss_source_urls
: Array[Text]
43 # Path to the log file
46 # Unique path of files names to prevent double download (may be empty)
47 var unique_pattern
: Array[Pattern]
49 # Exception where we ignore uniqueness and can be downloaded again (may be empty)
50 var unique_exceptions
: Array[Pattern]
52 # XML tag used for pattern recognition
53 fun tag_title
: String do return "title"
55 # XML tag of the link to act upon
56 fun tag_link
: String do return "link"
58 # Action to apply on each selected RSS element
59 fun act_on
(element
: Element)
61 var local_path
= download_destination_folder
.to_s
/ element
.title
62 element
.download_to
(local_path
)
66 # An element from an RSS feed
71 # Link to the file to download
74 redef fun to_s
do return "{title} @ {link}"
76 # Download this element to `path`
77 fun download_to
(path
: Text)
79 var request
= new CurlHTTPRequest(link
)
80 var response
= request
.download_to_file
(path
.to_s
)
82 if response
isa CurlResponseFailed then
83 sys
.stderr
.write
"Failed downloading URL '{link}' with: {response.error_msg} ({response.error_code})\n"
87 # Get an unique identifier for this element, uses `Config::unique_pattern`
88 fun unique_id
(config
: Config): String
90 for re
in config
.unique_pattern
do
91 var match
= title
.search
(re
)
93 return title
.substring
(0, match
.after
).to_lower
100 # Is this element except from uniqueness?
101 fun is_unique_exception
(config
: Config): Bool
103 for re
in config
.unique_exceptions
do
104 if title
.has
(re
) then
112 # Main program structure
117 # Local history (read from, them written to file)
118 var history
= new HashSet[Text]
123 # Read old log from file
124 if config
.log_path
.exists
then
125 var stream
= config
.log_path
.open_ro
126 history
.add_all stream
.read_all
.split
("\n")
130 # Get the pattern to search for
131 var patterns
= self.patterns
133 # Get all the elements from the RSS feeds
134 var elements
= new HashSet[Element]
135 for rss_url
in config
.rss_source_urls
do
136 var rss
= rss_url
.fetch_rss_content
137 elements
.add_all rss
.to_rss_elements
140 # Select the elements matching our pattern
141 var matches
= new HashSet[Element]
142 for pattern
in patterns
do for element
in elements
do
143 if element
.title
.has
(pattern
) then
149 print
"\n# {matches.length} matching elements:"
150 print matches
.join
("\n")
151 print
"\n# Downloading..."
154 for element
in matches
do
155 var unique_id
= element
.unique_id
(config
)
157 if history
.has
(unique_id
) then
158 # Do not download a file that is not unique according to `unique_id`
159 if not element
.is_unique_exception
(config
) then
160 # We make some exceptions
161 if sys
.verbose
then print
"File in log, skipping {element}"
167 if sys
.verbose
then print
"Acting on {element}"
169 tool_config
.act_on element
171 # Add `unique_id` to log
172 history
.add unique_id
175 # Save new log to file
176 var stream
= config
.log_path
.open_wo
177 for line
in history
do
184 # Gather all patterns from `Config::custom_patterns` and `Config::source_folder_path`
185 fun patterns
: Array[Pattern]
187 var patterns
= new Array[Pattern]
189 # Begin with custom pattern
190 for pattern_source
in config
.custom_patterns
do
191 patterns
.add pattern_source
194 # Get regex source from folder names
195 var folder_names
= new HashSet[Text]
196 for source_folder_path
in config
.regex_source_folders
do
197 var source_folder
= source_folder_path
199 if not source_folder
.exists
then
200 sys
.stderr
.write
"Regex source folder '{source_folder_path}' does not exists.\n"
204 for dir
in source_folder
.files
do if dir
.stat
.is_dir
then
205 folder_names
.add dir
.filename
209 # Compile our infered patterns
210 for folder_name
in folder_names
do
211 # Transform from "Some folder name" to "^Some.folder.name"
212 var regex_source
= folder_name
.
213 replace
(' ', ".").replace
('[', "\\[").replace
('(', "\\(").
214 replace
('+', "\\+").replace
('*', "\\*")
215 regex_source
= "^" + regex_source
217 var regex
= regex_source
.to_re
218 regex
.ignore_case
= true
223 if patterns
.is_empty
then
224 sys
.stderr
.write
"Do not have any pattern to work with.\n"
229 print
"# Generated {patterns.length} patterns"
230 print patterns
.join
("\n")
238 # Get the content of the RSS feed at `self`
239 fun fetch_rss_content
: Text
241 if sys
.verbose
then print
"\n# Downloading RSS file from '{self}'"
243 var request
= new CurlHTTPRequest(to_s
)
244 var response
= request
.execute
246 if response
isa CurlResponseSuccess then
247 var body
= response
.body_str
248 if sys
.verbose
then print
"Download successful"
250 else if response
isa CurlResponseFailed then
251 sys
.stderr
.write
"Failed downloading URL '{self}' with: {response.error_msg} ({response.error_code})\n"
258 # Get this RSS feed content as an `Array[Element]`
259 fun to_rss_elements
: Array[Element]
262 if xml
isa XMLError then
263 print_error
"RSS Parse Error: {xml.message}:{xml.location or else "null"}"
264 return new Array[Element]
266 var items
= xml
["rss"].first
["channel"].first
["item"]
268 var elements
= new Array[Element]
270 var title
= item
[tool_config
.tag_title
].first
.as(XMLStartTag).data
271 var link
= item
[tool_config
.tag_link
].first
.as(XMLStartTag).data
273 elements
.add
new Element(title
, link
)
277 print
"# Found elements:"
278 print elements
.join
("\n")
285 # Implement this method in your module to configure this tool
286 fun tool_config
: nullable Config do return null
290 print
"This tool is not configured, take a look at the example `sample_config.nit`"
292 abort # For the flow only
295 var tool
= new Downloader(c
)