var opt_out_src = new OptionString("Path to output source file (folder or file)", "--src", "-s")
var opt_assets = new OptionString("Path to assert dir where to put PNG files", "--assets", "-a")
-var opt_scale = new OptionFloat("Apply scaling to exported images (default at 1.0 of 90dpi)", 1.0, "--scale", "-x")
+var opt_scale = new OptionFloat("Apply scaling to exported images (default at 1.0 of 96dpi)", 1.0, "--scale", "-x")
var opt_gamnit = new OptionBool("Target the Gamnit framework (by default it targets Mnit)", "--gamnit", "-g")
var opt_pow2 = new OptionBool("Round the image size to the next power of 2", "--pow2")
var opt_help = new OptionBool("Print this help message", "--help", "-h")
# Output png file to assets
var png_path = "{assets_path}/images/{drawing_name}.png"
var proc2 = new Process.from_a(prog, [drawing, "--without-gui",
- "--export-dpi={(90.0*scale).to_i}",
+ "--export-dpi={(96.0*scale).to_i}",
"--export-png={png_path}",
"--export-area={min_x}:{y0}:{max_x}:{y1}",
"--export-background=#000000", "--export-background-opacity=0.0"])
if t.symbol == null then continue
# Check overlaps
- var tf = t.symbol.first
- var tl = t.symbol.last
+ var tf = t.symbol.as(not null).first
+ var tl = t.symbol.as(not null).last
if l != null and tf > l then continue
if tl != null and f > tl then continue
end
end
- # Produce a graphvis file for the automaton
- fun to_dot(filepath: String)
+ # Produce a graphviz string from the automatom
+ #
+ # Set `merge_transitions = false` to generate one edge by transition (default true).
+ fun to_dot(merge_transitions: nullable Bool): Writable
do
var names = new HashMap[State, String]
var ni = 0
ni += 1
end
- var f = new FileWriter.open(filepath)
- f.write("digraph g \{\n")
+ var f = new Buffer
+ f.append("digraph g \{\n")
+ f.append("rankdir=LR;")
+ var state_nb = 0
for s in states do
- f.write("s{names[s]}[shape=oval")
+ f.append("s{names[s]}[shape=circle")
#f.write("label=\"\",")
if accept.has(s) then
- f.write(",color=blue")
+ f.append(",shape=doublecircle")
end
if tags.has_key(s) then
- f.write(",label=\"")
+ f.append(",label=\"")
for token in tags[s] do
- f.write("{token.name.escape_to_c}\\n")
+ f.append("{token.name.escape_to_dot}\\n")
end
- f.write("\"")
+ f.append("\"")
else
- f.write(",label=\"\"")
+ f.append(",label=\"{state_nb}\"")
end
- f.write("];\n")
+ f.append("];\n")
var outs = new HashMap[State, Array[nullable TSymbol]]
for t in s.outs do
var a
for s2, a in outs do
var labe = ""
for c in a do
+ if merge_transitions == false then labe = ""
if not labe.is_empty then labe += "\n"
if c == null then
- labe += "''"
+ labe += "ε"
else
labe += c.to_s
end
+ if merge_transitions == false then
+ f.append("s{names[s]}->s{names[s2]} [label=\"{labe.escape_to_dot}\"];\n")
+ end
+ end
+ if merge_transitions == null or merge_transitions == true then
+ f.append("s{names[s]}->s{names[s2]} [label=\"{labe.escape_to_c}\"];\n")
end
- f.write("s{names[s]}->s{names[s2]} [label=\"{labe.escape_to_c}\"];\n")
end
+ state_nb += 1
end
- f.write("empty->s{names[start]}; empty[label=\"\",shape=none];\n")
-
- f.write("\}\n")
- f.close
+ f.append("empty->s{names[start]}; empty[label=\"\",shape=none];\n")
+ f.append("\}\n")
+ return f
end
# Transform a NFA to a DFA.
var dfa = new Automaton.empty
var n2d = new ArrayMap[Set[State], State]
- var seen = new ArraySet[Set[State]]
+ var seen = new ArraySet[Set[State]]
var alphabet = new HashSet[Int]
var st = eclosure([start])
var todo = [st]
seen.add(nfa_dest)
end
if lastst != null and lastst.to == dfa_dest then
- lastst.symbol.last = sym.last
+ lastst.symbol.as(not null).last = sym.last
else
lastst = dfa_state.add_trans(dfa_dest, sym)
end
for t in s.outs do
if t.symbol != null then continue
var to = t.to
- if res.has(to) then continue
+ if res.has(to) then continue
res.add(to)
todo.add(to)
end
var l = sym.last
if l != null and l < symbol then continue
var to = t.to
- if res.has(to) then continue
+ if res.has(to) then continue
res.add(to)
end
end
end
add "# Lexer generated by nitcc for the grammar {name}\n"
- add "module {name}_lexer is no_warning \"missing-doc\"\n"
+ add "module {name}_lexer is generated, no_warning \"missing-doc\"\n"
add("import nitcc_runtime\n")
var p = parser
var gram = autom.grammar
add "# Parser generated by nitcc for the grammar {name}"
- add "module {name}_parser is no_warning(\"missing-doc\",\"old-init\")"
+ add "module {name}_parser is generated, no_warning(\"missing-doc\",\"old-init\")"
add "import nitcc_runtime"
add "class Parser_{name}"
if not node isa NProd then
assert node isa NError
- print "{node.position.to_s} Syntax Error: {node.message}"
+ print "{node.position.as(not null)} Syntax Error: {node.message}"
exit 1
abort
end
var nfa = v2.nfa
print "NFA automaton: {nfa.states.length} states (see {name}.nfa.dot)"
-nfa.to_dot("{name}.nfa.dot")
+nfa.to_dot.write_to_file("{name}.nfa.dot")
var dfa = nfa.to_dfa.to_minimal_dfa
dfa.solve_token_inclusion
print "DFA automaton: {dfa.states.length} states (see {name}.dfa.dot)"
-dfa.to_dot("{name}.dfa.dot")
+dfa.to_dot.write_to_file("{name}.dfa.dot")
if dfa.tags.has_key(dfa.start) then
print "Error: Empty tokens {dfa.tags[dfa.start].join(" ")}"
f.write """# Generated by nitcc for the language {{{name}}}
# Standalone parser tester for the language {{{name}}}
-module {{{name}}}_test_parser
+module {{{name}}}_test_parser is generated
import nitcc_runtime
import {{{name}}}_lexer
import {{{name}}}_parser
end
is_building = false
var nre = self.children[2]
- res = nre.make_rfa
+ res = nre.make_nfa
nfa = res
return res
end
redef class Nre_id
# The named expression
var nexpr: nullable Nexpr
-
+
redef fun accept_check_name_visitor(v) do
var id = children.first.as(Nid)
var name = id.text
v.nexpr.precs.add(node)
end
- redef fun make_rfa
+ redef fun make_nfa
do
return nexpr.nfa.dup
end
redef class Node
# Build the NFA of the regular expression
- fun make_rfa: Automaton do
+ fun make_nfa: Automaton do
print inspect
abort
end
end
redef class Nstr
- redef fun value: String do return text.substring(1, text.length-2).unescape_nit
- redef fun make_rfa: Automaton
+ redef fun value do return text.substring(1, text.length-2).unescape_nit
+ redef fun make_nfa
do
var a = new Automaton.epsilon
for c in self.value.chars do
end
redef class Nch_dec
- redef fun value: String do return text.substring_from(1).to_i.code_point.to_s
- redef fun make_rfa: Automaton
+ redef fun value do return text.substring_from(1).to_i.code_point.to_s
+ redef fun make_nfa
do
var a = new Automaton.atom(self.value.chars.first.code_point)
return a
end
redef class Nch_hex
- redef fun value: String do return text.substring_from(2).to_hex.code_point.to_s
- redef fun make_rfa: Automaton
+ redef fun value do return text.substring_from(2).to_hex.code_point.to_s
+ redef fun make_nfa
do
var a = new Automaton.atom(self.value.chars.first.code_point)
return a
end
redef class NProd
- redef fun make_rfa: Automaton
+ redef fun make_nfa
do
- assert children.length == 1 else print "no make_rfa for {self}"
- return children.first.make_rfa
+ assert children.length == 1 else print "no make_nfa for {self}"
+ return children.first.make_nfa
end
end
redef class Nre_alter
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[0].make_rfa
- var b = children[2].make_rfa
+ var a = children[0].make_nfa
+ var b = children[2].make_nfa
a.alternate(b)
return a
end
end
redef class Nre_minus
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[0].make_rfa
- var b = children[2].make_rfa.to_dfa
+ var a = children[0].make_nfa
+ var b = children[2].make_nfa.to_dfa
for t in b.start.outs do
if not t.to.outs.is_empty then
# `b` is not a single char, so just use except
end
redef class Nre_end
- redef fun make_rfa
+ redef fun make_nfa
do
print "{children.first.position.to_s}: NOT YET IMPLEMENTED: token `End`; replaced with an empty string"
return new Automaton.epsilon
end
redef class Nre_and
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[0].make_rfa
+ var a = children[0].make_nfa
var ta = new Token("1")
a.tag_accept(ta)
- var b = children[2].make_rfa
+ var b = children[2].make_nfa
var tb = new Token("2")
b.tag_accept(tb)
end
redef class Nre_except
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[0].make_rfa
- var b = children[2].make_rfa
+ var a = children[0].make_nfa
+ var b = children[2].make_nfa
return a.except(b)
end
end
redef class Nre_shortest
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[2].make_rfa
+ var a = children[2].make_nfa
a = a.to_dfa
for s in a.accept do
for t in s.outs.to_a do t.delete
end
redef class Nre_longest
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[2].make_rfa
+ var a = children[2].make_nfa
a = a.to_dfa
for s in a.accept.to_a do
if not s.outs.is_empty then a.accept.remove(s)
end
redef class Nre_prefixes
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[2].make_rfa
+ var a = children[2].make_nfa
a.trim
a.accept.add_all a.states
return a
end
redef class Nre_conc
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[0].make_rfa
- var b = children[1].make_rfa
+ var a = children[0].make_nfa
+ var b = children[1].make_nfa
a.concat(b)
return a
end
end
redef class Nre_star
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[0].make_rfa
+ var a = children[0].make_nfa
a.close
return a
end
end
redef class Nre_ques
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[0].make_rfa
+ var a = children[0].make_nfa
a.optionnal
return a
end
end
redef class Nre_plus
- redef fun make_rfa
+ redef fun make_nfa
do
- var a = children[0].make_rfa
+ var a = children[0].make_nfa
a.plus
return a
end
end
redef class Nre_par
- redef fun make_rfa
+ redef fun make_nfa
do
- return children[1].make_rfa
+ return children[1].make_nfa
end
end
redef class Nre_class
- redef fun make_rfa: Automaton
+ redef fun make_nfa
do
var c1 = children[0].children[0].value
var c2 = children[3].children[0].value
end
redef class Nre_openclass
- redef fun make_rfa: Automaton
+ redef fun make_nfa
do
var c1 = children[0].children[0].value
if c1.length != 1 then
end
redef class Nre_any
- redef fun make_rfa: Automaton
+ redef fun make_nfa
do
var a = new Automaton.cla(0, null)
return a
# Lexer and its tokens.
# This file was generated by SableCC (http://www.sablecc.org/).
-module lexer is no_warning("missing-doc", "old-init")
+module lexer is generated, no_warning("missing-doc", "old-init")
intrude import parser_nodes
private import tables
# Parser.
# This file was generated by SableCC (http://www.sablecc.org/).
-module parser is no_warning("missing-doc", "old-init")
+module parser is generated, no_warning("missing-doc", "old-init")
intrude import parser_prod
import tables
# Raw AST node hierarchy.
# This file was generated by SableCC (http://www.sablecc.org/).
-package parser_abs
+package parser_abs is generated
import location
--- /dev/null
+*.dot
+*parser_parser.nit
+*parser_lexer.nit
+*_test_parser*
+*.out
+
+re_parser
+re_app
--- /dev/null
+NITC=../../bin/nitc
+NITCC=../../nitcc/src/nitcc
+NITUNIT=../../bin/nitunit
+
+all: re_parser re_app
+
+nitcc:
+ cd ../nitcc && make nitcc
+
+grammar: nitcc
+ cd src/ && ${NITCC} re_parser.sablecc
+
+re_parser: grammar
+ ${NITC} src/re_parser.nit
+
+re_app: grammar
+ ${NITC} src/re_app.nit
+
+check:
+ ${NITUNIT} .
+
+clean:
+ rm re_parser re_app 2>/dev/null || true
+ cd src/ && rm -r \
+ *.dot *.out \
+ re_parser_lexer.nit re_parser_parser.nit re_parser_test_parser.nit re_parser_parser_gen \
+ 2>/dev/null || true
--- /dev/null
+# RE Parser
+
+RE parser provides a simple API to regular expression parsing.
+It is also able to convert a regular expression into a NFA or a DFA and produce dot files from it.
+
+## Building RE parser
+
+From the `re_parser` directory:
+
+~~~raw
+make all
+~~~
+
+## RE parser in command line
+
+RE parser can be used as a command line tool to generate NFA and DFA dot files from a regular expression:
+
+~~~raw
+./re_parser "a(b|c)+d*"
+~~~
+
+Will produce the two files `nfa.dot` and `dfa.dot`.
+
+These can be directly viwed with `xdot`:
+
+~~~raw
+xdot nfa.dot
+xdot dfa.dot
+~~~
+
+Or translated to png images with `dot`:
+
+~~~raw
+dot -Tpng -onfa.png nfa.dot
+dot -Tpng -odfa.png dfa.dot
+~~~
+
+See `man dot` for available formats.
+
+## RE parser as a web app
+
+RE parser comes with a web app that allow users to submit regular expression and see the resulting NFA and DFA.
+
+To run the web app server:
+
+~~~raw
+./re_app --host localhost --port 3000
+~~~
+
+The server will be available at <a href='http://localhost:3000'>http://localhost:3000</a>.
+
+## RE parser as a library
+
+You can also use RE parser as a library by importing `re_parser`.
+
+ import re_parser
+
+ var re = "a(b|c)+d*"
+
+ # Parse re
+ var re_parser = new REParser
+ var node = re_parser.parse_re(re)
+
+ if node == null then
+ print re_parser.last_error.as(not null)
+ exit 1
+ abort
+ end
+
+ # Build NFA and DFA
+ var nfa = re_parser.make_nfa(node)
+ print nfa.to_dot(false)
+ print nfa.to_dfa.to_dot(false)
+
+Use `to_dot(true)` to merge transitions on characters.
--- /dev/null
+[package]
+name=re_parser
+tags=re
+maintainer=Alexandre Terrasa <alexandre@moz-code.org>
+license=Apache-2.0
+[upstream]
+browse=https://github.com/nitlang/nit/tree/master/contrib/re_parser/
+git=https://github.com/nitlang/nit.git
+git.directory=contrib/re_parser/
+homepage=http://nitlanguage.org
+issues=https://github.com/nitlang/nit/issues
--- /dev/null
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module re_app
+
+import re_parser
+import popcorn
+import popcorn::pop_config
+
+class REHandler
+ super Handler
+
+ redef fun get(req, res) do
+ res.html new RETemplate
+ end
+
+ redef fun post(req, res) do
+ # TODO Retrieve re from post body
+ var re = req.string_arg("re")
+
+ var tpl = new RETemplate
+
+ if re == null or re.is_empty then
+ tpl.error = "Error: empty regexp"
+ res.html tpl
+ return
+ end
+
+ tpl.re = re
+
+ # Parse re
+ var re_parser = new REParser
+ var node = re_parser.parse_re(re)
+
+ if node == null then
+ tpl.error = re_parser.last_error.as(not null)
+ res.html tpl
+ return
+ end
+
+ # Build nfa and dfa
+ var nfa = re_parser.make_nfa(node)
+ tpl.nfa = automaton_to_svg(nfa)
+ tpl.dfa = automaton_to_svg(nfa.to_dfa)
+
+ res.html tpl
+ end
+
+ private fun automaton_to_svg(automaton: Automaton): String do
+ automaton.to_dot(false).write_to_file "dot.tmp"
+ sys.system "dot -Tsvg -osvg.tmp dot.tmp"
+ var svg = "svg.tmp".to_path.read_all
+ sys.system "rm -f dot.tmp svg.tmp"
+ return svg
+ end
+end
+
+class RETemplate
+ super Template
+
+ var re = "a*(b|cd?)+"
+ var nfa: nullable String = null
+ var dfa: nullable String = null
+ var error: nullable String = null
+
+ redef fun rendering do
+ add """
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8" />
+ <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
+ <title>REParser</title>
+ <link rel="stylesheet"
+ href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" />
+ </head>
+ <body>
+ <div class="container">
+ <h1>Regular Expressions to NFA and DFA</h1>
+ <br><br>
+ <label for="re">Enter a regular expression:</label>
+ <form action="/" method="POST" class="input-group">
+ <input id="re" name="re" type="text" class="form-control" value="{{{re}}}">
+ <span class="input-group-btn">
+ <input type="submit" class="btn btn-default" value="Convert" />
+ </span>
+ </form>
+ <br><br>"""
+
+ var error = self.error
+ if error != null then
+ add """<p class="text-danger">{{{error}}}</p>"""
+ end
+
+ var nfa = self.nfa
+ if nfa != null then
+ add """
+ <h2>NFA</h2>
+ <div>{{{nfa}}}</div><br><br>"""
+ end
+
+ var dfa = self.dfa
+ if dfa != null then
+ add """
+ <h2>DFA</h2>
+ <div>{{{dfa}}}</div><br><br>"""
+ end
+
+ add """</div>
+ <div class="text-center text-muted">
+ <p>Powered by <a href="http://nitlanguage.org">nit</a>!</p>
+ </div>
+ </body>
+</html>"""
+ end
+end
+
+var config = new AppConfig
+config.parse_options(args)
+
+var app = new App
+app.use("/", new REHandler)
+app.listen(config.app_host, config.app_port)
--- /dev/null
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module re_parser
+
+import nitcc::autom
+import re_parser_lexer
+import re_parser_parser
+
+# Parse regular expression into NFA
+#
+# ~~~
+# # Parse the regular expression
+# var re = "(a|b)*"
+# var re_parser = new REParser
+# var node = re_parser.parse_re(re)
+#
+# # Check syntax errors
+# assert node != null else
+# print re_parser.last_error.as(not null)
+# end
+#
+# # Build nfa and dfa
+# var nfa = re_parser.make_nfa(node)
+# print nfa.to_dot
+# print nfa.to_dfa.to_dot
+# ~~~
+class REParser
+
+ # Parse the regular expression `re` and return the root production node.
+ #
+ # Returns `null` in case of syntax error. See `last_error`.
+ #
+ # ~~~
+ # var re_parser = new REParser
+ #
+ # # Valid regular expression
+ # assert re_parser.parse_re("(a|b)*") != null
+ # assert re_parser.last_error == null
+ #
+ # # Invalid regular expression
+ # assert re_parser.parse_re("a|b)*") == null
+ # assert re_parser.last_error != null
+ # ~~~
+ fun parse_re(re: String): nullable NProd do
+ var l = new Lexer_re_parser(re)
+ var ts = l.lex
+
+ var p = new Parser_re_parser
+ p.tokens.add_all ts
+
+ var node = p.parse
+
+ if not node isa NProd then
+ if node isa NError then
+ last_error = "{node.position.as(not null).to_s} Syntax Error: {node.message}"
+ else
+ last_error = "Parsing Error: expected `NProd` got `{node.class_name}`"
+ end
+ return null
+ end
+ last_error = null
+ return node
+ end
+
+ # Contains the error from the last call to `parse_re` or null if no error
+ #
+ # ~~~
+ # var re_parser = new REParser
+ #
+ # # Invalid regular expression
+ # assert re_parser.parse_re("a|b)*") == null
+ # assert re_parser.last_error != null
+ # ~~~
+ var last_error: nullable String
+
+ # Build the NFA for `node`
+ #
+ # Use `parse_re` to transform a re string into a NProd.
+ fun make_nfa(node: NProd): Automaton do
+ var v = new REVisitor
+ v.start(node)
+ return v.nfa
+ end
+end
+
+class REVisitor
+ super Visitor
+
+ var nfa = new Automaton
+
+ fun start(n: Node) do
+ enter_visit(n)
+ end
+
+ redef fun visit(n) do n.accept_revisitor(self)
+end
+
+redef class Node
+ fun accept_revisitor(v: REVisitor) do visit_children(v)
+
+ # Build the NFA of the regular expression
+ fun make_nfa: Automaton do
+ print inspect
+ abort
+ end
+end
+
+redef class Nre
+ redef fun accept_revisitor(v) do
+ v.nfa = make_nfa
+ end
+
+ redef fun make_nfa do
+ var a = new Automaton
+ for child in children do
+ if child == null then continue
+ a.concat(child.make_nfa)
+ end
+ return a
+ end
+end
+
+redef class Nre_char
+ redef fun make_nfa do
+ return new Automaton.atom(n_char.text.chars.first.code_point)
+ end
+end
+
+redef class Nre_alter
+ redef fun make_nfa
+ do
+ var a = n_re.make_nfa
+ var b = n_re2.make_nfa
+ a.alternate(b)
+ return a
+ end
+end
+
+redef class Nre_conc
+ redef fun make_nfa
+ do
+ var a = n_re.make_nfa
+ var b = n_re2.make_nfa
+ a.concat(b)
+ return a
+ end
+end
+
+redef class Nre_star
+ redef fun make_nfa
+ do
+ var a = n_re.make_nfa
+ a.close
+ return a
+ end
+end
+
+redef class Nre_ques
+ redef fun make_nfa
+ do
+ var a = n_re.make_nfa
+ a.optionnal
+ return a
+ end
+end
+
+redef class Nre_plus
+ redef fun make_nfa
+ do
+ var a = n_re.make_nfa
+ a.plus
+ return a
+ end
+end
+
+redef class Nre_par
+ redef fun make_nfa
+ do
+ return n_re.make_nfa
+ end
+end
+
+# Parse arguments
+if args.is_empty then
+ print "usage: re_parser <re>"
+ exit 1
+end
+var re = args.first
+
+# Parse re
+var re_parser = new REParser
+var node = re_parser.parse_re(re)
+
+if node == null then
+ print re_parser.last_error.as(not null)
+ exit 1
+ abort
+end
+
+# Build nfa and dfa
+var nfa = re_parser.make_nfa(node)
+nfa.to_dot(false).write_to_file("nfa.dot")
+nfa.to_dfa.to_dot(false).write_to_file("dfa.dot")
+print "Produced files `nfa.dot` and `dfa.dot`"
--- /dev/null
+Grammar re_parser;
+
+Lexer
+
+// A printable character (inside strings)
+char = ' ' ...;
+
+// Igndored stufs
+blank = ' ';
+
+Parser
+
+Ignored blank;
+
+re =
+ {char:} char |
+ {par:} '(' re ')'
+Unary
+ {ques:} re '?' |
+ {star:} re '*' |
+ {plus:} re '+'
+Left
+ {conc:} re re
+Left
+ {alter:} re '|' re;
#include <stdio.h>
#include <unistd.h>
#include <signal.h>
-#ifndef _WIN32
+ #include <sys/types.h>
+
+#ifdef _WIN32
+ #include <windows.h>
+ #include <fcntl.h>
+#else
#include <sys/wait.h>
#endif
-`}
-
-in "C Header" `{
- #include <sys/types.h>
- // FIXME this should be in the "C" block when bug on module blocks is fixed
- // or, even better, replace the C structure by a Nit object.
typedef struct se_exec_data se_exec_data_t;
struct se_exec_data {
+#ifdef _WIN32
+ HANDLE h_process;
+ HANDLE h_thread;
+#endif
pid_t id;
int running;
int status;
# Internal code to handle execution
protected fun execute
do
- # Pass the arguments as a big C string where elements are separated with '\0'
- var args = new FlatBuffer
- var l = 1 # Number of elements in args
- args.append(command)
var arguments = self.arguments
- if arguments != null then
- for a in arguments do
- args.add('\0')
- args.append(a)
+
+ var args = new FlatBuffer
+ var argc = 1
+
+ if not is_windows then
+ # Pass the arguments as a big C string where elements are separated with '\0'
+ args.append command
+ if arguments != null then
+ for a in arguments do
+ args.add '\0'
+ args.append a
+ end
+ argc += arguments.length
+ end
+ else
+ # Combine the program and args in a single string
+ assert not command.chars.has('"')
+ args = new FlatBuffer
+
+ args.add '"'
+ args.append command
+ args.add '"'
+
+ if arguments != null then
+ for a in arguments do
+ args.append " \""
+ args.append a.replace('"', "\\\"")
+ args.add '"'
+ end
end
- l += arguments.length
end
- data = basic_exec_execute(command.to_cstring, args.to_s.to_cstring, l, pipeflags)
+
+ data = basic_exec_execute(command.to_cstring, args.to_s.to_cstring, argc, pipeflags)
+ assert not data.address_is_null else print_error "Internal error executing: {command}"
end
private var data: NativeProcess
+
private fun basic_exec_execute(prog, args: CString, argc: Int, pipeflag: Int): NativeProcess `{
#ifdef _WIN32
- // FIXME use a higher level abstraction to support WIN32
- return -1;
+ SECURITY_ATTRIBUTES sec_attr;
+ sec_attr.nLength = sizeof(SECURITY_ATTRIBUTES);
+ sec_attr.bInheritHandle = TRUE;
+ sec_attr.lpSecurityDescriptor = NULL;
+
+ STARTUPINFO start_info;
+ ZeroMemory(&start_info, sizeof(STARTUPINFO));
+ start_info.cb = sizeof(STARTUPINFO);
+ start_info.dwFlags = STARTF_USESTDHANDLES;
+
+ HANDLE in_fd[2];
+ HANDLE out_fd[2];
+ HANDLE err_fd[2];
+
+ se_exec_data_t *result = (se_exec_data_t*)malloc(sizeof(se_exec_data_t));
+
+ // Redirect stdin?
+ if (pipeflag & 1) {
+ if (!CreatePipe(&in_fd[0], &in_fd[1], &sec_attr, 0)) {
+ return NULL;
+ }
+ start_info.hStdInput = in_fd[0];
+ result->in_fd = _open_osfhandle((intptr_t)in_fd[1], _O_APPEND);
+ if ( !SetHandleInformation(in_fd[1], HANDLE_FLAG_INHERIT, 0) )
+ return NULL;
+ } else {
+ start_info.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
+ result->in_fd = -1;
+ }
+
+ // Redirect stdout?
+ if (pipeflag & 2) {
+ if (!CreatePipe(&out_fd[0], &out_fd[1], &sec_attr, 0)) {
+ return NULL;
+ }
+ start_info.hStdOutput = out_fd[1];
+ result->out_fd = _open_osfhandle((intptr_t)out_fd[0], _O_RDONLY);
+ if ( !SetHandleInformation(out_fd[0], HANDLE_FLAG_INHERIT, 0) )
+ return NULL;
+ } else {
+ start_info.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE);
+ result->out_fd = -1;
+ }
+
+ // Redirect stderr?
+ if (pipeflag & 4) {
+ if (!CreatePipe(&err_fd[0], &err_fd[1], &sec_attr, 0)) {
+ return NULL;
+ }
+ start_info.hStdError = err_fd[1];
+ result->err_fd = _open_osfhandle((intptr_t)err_fd[0], _O_RDONLY);
+ if ( !SetHandleInformation(err_fd[0], HANDLE_FLAG_INHERIT, 0) )
+ return NULL;
+ } else {
+ start_info.hStdError = GetStdHandle(STD_ERROR_HANDLE);
+ result->err_fd = -1;
+ }
+
+ PROCESS_INFORMATION proc_info;
+ ZeroMemory(&proc_info, sizeof(PROCESS_INFORMATION));
+
+ BOOL created = CreateProcess(NULL,
+ args, // command line
+ NULL, // process security attributes
+ NULL, // primary thread security attributes
+ TRUE, // inherit handles
+ 0, // creation flags
+ NULL, // use parent's environment
+ NULL, // use parent's current directory
+ &start_info,
+ &proc_info);
+
+ // Error?
+ if (!created) {
+ result->running = 0;
+ result->status = 127;
+
+ // Close subprocess pipes
+ if (pipeflag & 1) CloseHandle(in_fd[1]);
+ if (pipeflag & 2) CloseHandle(out_fd[0]);
+ if (pipeflag & 3) CloseHandle(err_fd[0]);
+ } else {
+ result->h_process = proc_info.hProcess;
+ result->h_thread = proc_info.hThread;
+ result->id = GetProcessId(proc_info.hProcess);
+ result->running = 1;
+ }
+
+ return result;
#else
se_exec_data_t* result = NULL;
int id;
private extern class NativeProcess `{ se_exec_data_t* `}
- fun id: Int `{ return self->id; `}
+ fun id: Int `{ return (long)self->id; `}
fun status: Int `{ return self->status; `}
fun in_fd: Int `{ return self->in_fd; `}
fun out_fd: Int `{ return self->out_fd; `}
fun err_fd: Int `{ return self->err_fd; `}
fun is_finished: Bool `{
-#ifdef _WIN32
- // FIXME use a higher level abstraction to support WIN32
- return 0;
-#else
int result = (int)0;
- int status;
if (self->running) {
+#ifdef _WIN32
+ if (WaitForSingleObject(self->h_process, 0) == 0) {
+ /* child is finished */
+ result = 1;
+
+ long unsigned int status;
+ GetExitCodeProcess(self->h_process, &status);
+ self->running = 0;
+ self->status = (int)status;
+
+ CloseHandle(self->h_process);
+ CloseHandle(self->h_thread);
+ }
+#else
+ int status;
int id = waitpid(self->id, &status, WNOHANG);
if (id != 0) {
/* child is finished */
self->status = WEXITSTATUS(status);
self->running = 0;
}
+#endif
}
else{
result = (int)1;
}
return result;
-#endif
`}
fun wait `{
-#ifndef _WIN32
- // FIXME use a higher level abstraction to support WIN32
+#ifdef _WIN32
+ long unsigned int status;
+ if (self->running) {
+ WaitForSingleObject(self->h_process, INFINITE);
+ GetExitCodeProcess(self->h_process, &status);
+
+ CloseHandle(self->h_process);
+ CloseHandle(self->h_thread);
+
+ self->status = (int)status;
+ self->running = 0;
+ }
+#else
int status;
if (self->running) {
waitpid(self->id, &status, 0);
# ~~~
fun simplify_path: String
do
- var a = self.split_with("/")
+ var path_sep = if is_windows then "\\" else "/"
+ var a = self.split_with(path_sep)
var a2 = new Array[String]
for x in a do
if x == "." and not a2.is_empty then continue # skip `././`
`}
fun io_write(buf: CString, from, len: Int): Int `{
- return fwrite(buf+from, 1, len, self);
+ size_t res = fwrite(buf+from, 1, len, self);
+#ifdef _WIN32
+ // Force flushing buffer because end of line does not trigger a flush
+ fflush(self);
+#endif
+ return (long)res;
`}
fun write_byte(value: Byte): Int `{
class StringWriter
super Writer
- private var content = new Array[String]
- redef fun to_s do return content.plain_to_s
+ private var content = new Buffer
+ redef fun to_s do return content.to_s
redef fun is_writable do return not closed
redef fun write_bytes(b) do
- content.add(b.to_s)
+ content.append(b.to_s)
end
redef fun write(str)
do
assert not closed
- content.add(str.to_s)
+ content.append(str)
+ end
+
+ redef fun write_char(c)
+ do
+ assert not closed
+ content.add(c)
end
# Is the stream closed?
var array = get("/repos/{repo.full_name}/branches")
var res = new Array[Branch]
if not array isa JsonArray then return res
- return deserialize(array.to_json).as(Array[Branch])
+ var deser = deserialize(array.to_json)
+ if deser isa Array[Object] then return res # empty array
+ return deser.as(Array[Branch])
end
# List of issues associated with their ids.
# Provides access to [Github user data](https://developer.github.com/v3/users/).
# Should be accessed from `GithubAPI::load_user`.
class User
- super GithubEntity
+ super GitUser
serialize
# Github login.
var parents: nullable Array[Commit] = null is writable
# Author of the commit.
- var author: nullable User is writable
+ var author: nullable GitUser is writable
# Committer of the commit.
- var committer: nullable User is writable
+ var committer: nullable GitUser is writable
# Authoring date as String.
var author_date: nullable String is writable
# Commit message.
var message: nullable String is writable
+
+ # Git commit representation linked to this commit.
+ var commit: nullable GitCommit
+end
+
+# A Git Commit representation
+class GitCommit
+ super GithubEntity
+ serialize
+
+ # Commit SHA.
+ var sha: nullable String is writable
+
+ # Parent commits of `self`.
+ var parents: nullable Array[GitCommit] = null is writable
+
+ # Author of the commit.
+ var author: nullable GitUser is writable
+
+ # Committer of the commit.
+ var committer: nullable GitUser is writable
+
+ # Commit message.
+ var message: nullable String is writable
+end
+
+# Git user authoring data
+class GitUser
+ super GithubEntity
+ serialize
+
+ # Authoring date.
+ var date: nullable String = null is writable
+
+ # Authoring date as ISODate.
+ fun iso_date: nullable ISODate do
+ var date = self.date
+ if date == null then return null
+ return new ISODate.from_string(date)
+ end
end
# A Github issue.
var closed_by: nullable User is writable
# Is this issue linked to a pull request?
- var is_pull_request: Bool = false is writable, noserialize
+ var is_pull_request: Bool = false is writable
end
# A Github pull request.
serialize
# The milestone id on Github.
- var number: Int is writable
+ var number: nullable Int = null is writable
# Milestone title.
var title: String is writable
# Milestone long description.
- var description: String is writable
+ var description: nullable String is writable
# Count of opened issues linked to this milestone.
- var open_issues: Int is writable
+ var open_issues: nullable Int = null is writable
# Count of closed issues linked to this milestone.
- var closed_issues: Int is writable
+ var closed_issues: nullable Int = null is writable
# Milestone state.
- var state: String is writable
+ var state: nullable String is writable
# Creation time as String.
- var created_at: String is writable
+ var created_at: nullable String is writable
# Creation time as ISODate.
- fun iso_created_at: nullable ISODate do return new ISODate.from_string(created_at)
+ fun iso_created_at: nullable ISODate do
+ var created_at = self.created_at
+ if created_at == null then return null
+ return new ISODate.from_string(created_at)
+ end
# User that created this milestone.
- var creator: User is writable
+ var creator: nullable User is writable
# Due time as String (if any).
var due_on: nullable String is writable
super JsonDeserializer
redef fun class_name_heuristic(json_object) do
- if json_object.has_key("login") or json_object.has_key("email") then
+ if json_object.has_key("login") then
return "User"
else if json_object.has_key("full_name") then
return "Repo"
return "Branch"
else if json_object.has_key("sha") and json_object.has_key("ref") then
return "PullRef"
- else if json_object.has_key("sha") or (json_object.has_key("id") and json_object.has_key("tree_id")) then
+ else if (json_object.has_key("sha") and json_object.has_key("commit")) or (json_object.has_key("id") and json_object.has_key("tree_id")) then
return "Commit"
+ else if json_object.has_key("sha") and json_object.has_key("tree") then
+ return "GitCommit"
+ else if json_object.has_key("name") and json_object.has_key("date") then
+ return "GitUser"
else if json_object.has_key("number") and json_object.has_key("patch_url") then
return "PullRequest"
else if json_object.has_key("open_issues") and json_object.has_key("closed_issues") then
issue.is_pull_request = true
end
return issue
+ else if name == "Commit" then
+ var commit = super.as(Commit)
+ var git_commit = commit.commit
+ if git_commit != null then commit.message = git_commit.message
+ return commit
end
return super
end
if response isa CurlResponseSuccess then
var obj = response.body_str.parse_json
if obj isa JsonObject then
- if obj.keys.has("message") then
+ if obj.keys.has("message") and obj.keys.has("documentation_url") then
print "Message from Github API: {obj["message"] or else ""}"
print "Requested URI: {uri}"
abort
if response isa CurlResponseSuccess then
var obj = response.body_str.parse_json
if obj isa JsonObject then
- if obj.keys.has("message") then
+ if obj.keys.has("message") and obj.keys.has("documentation_url") then
var title = "GithubAPIError"
var msg = obj["message"].as(not null).to_s
var err = new GithubError(msg, title)
--- /dev/null
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Copyright 2016 Alexandre Terrasa <alexandre@moz-code.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module loader
+
+import config
+import github::wallet
+import github::events
+import popcorn::pop_repos
+import popcorn::pop_logging
+
+# Loader configuration file
+class LoaderConfig
+ super IniConfig
+
+ redef var default_config_file = "loader.ini"
+
+ # Default database host string for MongoDb
+ var default_db_host = "mongodb://localhost:27017/"
+
+ # Default database hostname
+ var default_db_name = "github_loader"
+
+ # MongoDb host name
+ var opt_db_host = new OptionString("MongoDb host", "--db-host")
+
+ # MongoDb database name
+ var opt_db_name = new OptionString("MongoDb database name", "--db-name")
+
+ # --verbose
+ var opt_verbose = new OptionCount("Verbosity level", "-v", "--verbose")
+
+ # --no-colors
+ var opt_no_colors = new OptionBool("Do not use colors in output", "--no-colors")
+
+ # --tokens
+ var opt_tokens = new OptionArray("Token list", "--tokens")
+
+ # --show-wallet
+ var opt_show_wallet = new OptionBool("Show wallet status", "--show-wallet")
+
+ # --show-jobs
+ var opt_show_jobs = new OptionBool("Show jobs status", "--show-jobs")
+
+ # --no-branches
+ var opt_no_branches = new OptionBool("Do not load branches", "--no-branches")
+
+ # --no-commits
+ var opt_no_commits = new OptionBool("Do not load commits from default branch", "--no-commits")
+
+ # --no-issues
+ var opt_no_issues = new OptionBool("Do not load issues", "--no-issues")
+
+ # --no-comments
+ var opt_no_comments = new OptionBool("Do not load issue comments", "--no-comments")
+
+ # --no-events
+ var opt_no_events = new OptionBool("Do not load issues events", "--no-events")
+
+ # --from
+ var opt_start = new OptionInt("Start loading issues from a number", 0, "--from")
+
+ # --clear
+ var opt_clear = new OptionBool("Clear job for given repo name", "--clear")
+
+ init do
+ super
+ tool_description = "Usage: loader <repo_name>"
+ add_option(opt_db_host, opt_db_name)
+ add_option(opt_tokens, opt_show_wallet)
+ add_option(opt_verbose, opt_no_colors)
+ add_option(opt_show_jobs, opt_no_commits, opt_no_issues, opt_no_comments, opt_no_events)
+ add_option(opt_start, opt_clear)
+ end
+
+ # MongoDB server used for data persistence
+ fun db_host: String do
+ return opt_db_host.value or else ini["db.host"] or else default_db_host
+ end
+
+ # MongoDB DB used for data persistence
+ fun db_name: String do
+ return opt_db_name.value or else ini["db.name"] or else default_db_name
+ end
+
+ # Mongo db client
+ var client = new MongoClient(db_host) is lazy
+
+ # Mongo db instance
+ var db: MongoDb = client.database(db_name) is lazy
+
+ # Github tokens used to access data.
+ var tokens: Array[String] is lazy do
+ var arr = opt_tokens.value
+ if arr.is_empty then
+ var iarr = ini.at("tokens")
+ if iarr != null then arr = iarr.values.to_a
+ end
+ return arr or else new Array[String]
+ end
+
+ # Github tokens wallet\13
+ var wallet: GithubWallet is lazy do
+ var wallet = new GithubWallet.from_tokens(tokens)
+ wallet.no_colors = no_colors
+ return wallet
+ end
+
+ # Use colors in console display
+ fun no_colors: Bool do
+ if opt_no_colors.value then return true
+ return ini["loader.no_colors"] == "true"
+ end
+
+ # Verbosity level (the higher the more verbose)
+ fun verbose_level: Int do
+ var opt = opt_start.value
+ if opt > 0 then return opt
+ var v = ini["loader.verbose"]
+ if v != null then return v.to_i
+ return 4
+ end
+
+ # Logger used to print things
+ var logger: ConsoleLog is lazy do
+ var logger = new ConsoleLog
+ logger.level = verbose_level
+ return logger
+ end
+
+ # Should we avoid loading branches?
+ fun no_branches: Bool do
+ if opt_no_branches.value then return true
+ return ini["loader.no_branches"] == "true"
+ end
+
+ # Should we avoid loading commits?
+ fun no_commits: Bool do
+ if opt_no_commits.value then return true
+ return ini["loader.no_commits"] == "true"
+ end
+
+ # Should we avoid loading issues?
+ fun no_issues: Bool do
+ if opt_no_issues.value then return true
+ return ini["loader.no_issues"] == "true"
+ end
+
+ # Should we avoid loading issue comments?
+ fun no_comments: Bool do
+ if opt_no_comments.value then return true
+ return ini["loader.no_comments"] == "true"
+ end
+
+ # Should we avoid loading events?
+ fun no_events: Bool do
+ if opt_no_events.value then return true
+ return ini["loader.no_events"] == "true"
+ end
+
+ # At which issue number should we start?
+ fun start_from_issue: Int do
+ var opt = opt_start.value
+ if opt > 0 then return opt
+ var v = ini["loader.start"]
+ if v != null then return v.to_i
+ return 1
+ end
+end
+
+redef class GithubWallet
+ redef fun api do
+ var api = super
+ api.enable_cache = true
+ return api
+ end
+end
+
+class Loader
+
+ var config = new LoaderConfig
+
+ # Jobs repository
+ var jobs: LoaderJobRepo is lazy do
+ return new LoaderJobRepo(config.db.collection("loader_status"))
+ end
+
+ var repos: RepoRepo is lazy do
+ return new RepoRepo(config.db.collection("repos"))
+ end
+
+ var branches: BranchRepo is lazy do
+ return new BranchRepo(config.db.collection("branches"))
+ end
+
+ var commits: CommitRepo is lazy do
+ return new CommitRepo(config.db.collection("commits"))
+ end
+
+ var issues: IssueRepo is lazy do
+ return new IssueRepo(config.db.collection("issues"))
+ end
+
+ var pulls: PullRequestRepo is lazy do
+ return new PullRequestRepo(config.db.collection("pull_requests"))
+ end
+
+ var issue_comments: IssueCommentRepo is lazy do
+ return new IssueCommentRepo(config.db.collection("issue_comments"))
+ end
+
+ var issue_events: IssueEventRepo is lazy do
+ return new IssueEventRepo(config.db.collection("issue_events"))
+ end
+
+ fun start(repo_full_name: String) do
+ var job = jobs.find_by_id(repo_full_name)
+ if job == null then
+ log.info "Creating new job for `{repo_full_name}`"
+ job = add_job(repo_full_name)
+ else
+ log.info "Resuming pending job for `{repo_full_name}`"
+ end
+ print "Load history for {job}..."
+ load_branches(job)
+ load_issues(job)
+ finish_job(job)
+ end
+
+ fun remove(repo_full_name: String) do
+ var job = jobs.find_by_id(repo_full_name)
+ if job == null then
+ log.info "No job found for `{repo_full_name}`"
+ else
+ jobs.remove_by_id(repo_full_name)
+ log.info "Deleted job for `{repo_full_name}`"
+ end
+ end
+
+ # Show wallet status
+ fun show_wallet do config.wallet.show_status
+
+ # Show jobs status
+ fun show_jobs do
+ var jobs = jobs.find_all
+ print "{jobs.length} jobs pending..."
+ for job in jobs do
+ print " * {job}"
+ end
+ print "\nUse `loader <job> to start a new or resume a pending one"
+ end
+
+ # Add a new job
+ fun add_job(repo_full_name: String): LoaderJob do
+ var repo = config.wallet.api.load_repo(repo_full_name)
+ assert repo != null else
+ error "Repository `{repo_full_name}` not found"
+ end
+ repos.save repo
+ var job = new LoaderJob(repo, config.start_from_issue)
+ jobs.save job
+ return job
+ end
+
+ # Finish a job
+ fun finish_job(job: LoaderJob) do
+ print "Finished job {job}"
+ jobs.remove_by_id(job.id)
+ end
+
+ fun load_branches(job: LoaderJob) do
+ if config.no_branches then return
+
+ var api = config.wallet.api
+ var repo = job.repo
+ for branch in api.load_repo_branches(repo) do
+ branch.repo = repo
+ branches.save branch
+ load_commits(job, branch)
+ end
+ end
+
+ fun load_commits(job: LoaderJob, branch: Branch) do
+ if config.no_commits then return
+ load_commit(job, branch.commit.sha)
+ end
+
+ fun load_commit(job: LoaderJob, commit_sha: String) do
+ if commits.find_by_id(commit_sha) != null then return
+ var api = config.wallet.api
+ var commit = api.load_commit(job.repo, commit_sha)
+ # print commit or else "NULL"
+ if commit == null then return
+ var message = commit.message or else "no message"
+ log.info "Load commit {commit_sha}: {message.split("\n").first}"
+ commit.repo = job.repo
+ commits.save commit
+ var parents = commit.parents
+ if parents == null then return
+ for parent in parents do
+ load_commit(job, parent.sha)
+ end
+ end
+
+ # Load game for `repo_name`.
+ fun load_issues(job: LoaderJob) do
+ if config.no_issues then return
+
+ var i = job.last_issue
+ var last_issue = load_last_issue(job)
+ if last_issue != null then
+ while i <= last_issue.number do
+ load_issue(job, i)
+ job.last_issue = i
+ jobs.save job
+ i += 1
+ end
+ end
+ end
+
+ # Load the `repo` last issue or abort.
+ private fun load_last_issue(job: LoaderJob): nullable Issue do
+ var api = config.wallet.api
+ return api.load_repo_last_issue(job.repo)
+ end
+
+ # Load an issue or abort.
+ private fun load_issue(job: LoaderJob, issue_number: Int) do
+ if issues.find_by_id("{job.repo.mongo_id}/{issue_number}") != null then return
+
+ var api = config.wallet.api
+ var issue = api.load_issue(job.repo, issue_number)
+ assert issue != null else
+ check_error(api, "Issue #{issue_number} not found")
+ end
+ if issue.is_pull_request then
+ load_pull(job, issue)
+ else
+ log.info "Load issue #{issue.number}: {issue.title.split("\n").first}"
+ issue.repo = job.repo
+ issues.save issue
+ load_issue_events(job, issue)
+ end
+ load_issue_comments(job, issue)
+ end
+
+ # Load issue comments.
+ private fun load_issue_comments(job: LoaderJob, issue: Issue) do
+ if config.no_comments then return
+ var api = config.wallet.api
+ for comment in api.load_issue_comments(job.repo, issue) do
+ comment.repo = job.repo
+ issue_comments.save comment
+ end
+ end
+
+ # Load issue events.
+ private fun load_issue_events(job: LoaderJob, issue: Issue) do
+ if config.no_events then return
+
+ var api = config.wallet.api
+ for event in api.load_issue_events(job.repo, issue) do
+ event.repo = job.repo
+ issue_events.save event
+ end
+ end
+
+ # Load a pull request or abort.
+ private fun load_pull(job: LoaderJob, issue: Issue): PullRequest do
+ var api = config.wallet.api
+ var pr = api.load_pull(job.repo, issue.number)
+ assert pr != null else
+ check_error(api, "Pull request #{issue.number} not found")
+ end
+ log.info "Load pull request #{issue.number}: {pr.title.split("\n").first}"
+ pr.repo = job.repo
+ pulls.save pr
+ load_pull_events(job, pr)
+ return pr
+ end
+
+ # Load pull events.
+ private fun load_pull_events(job: LoaderJob, pull: PullRequest) do
+ if config.no_events then return
+
+ var api = config.wallet.api
+ for event in api.load_issue_events(job.repo, pull) do
+ event.repo = job.repo
+ issue_events.save event
+ end
+ end
+
+ # Check if the API is in error state then abort
+ fun check_error(api: GithubAPI, message: nullable String) do
+ var err = api.last_error
+ if err != null then
+ error message or else err.message
+ end
+ end
+
+ # Logger shortcut
+ fun log: ConsoleLog do return config.logger
+
+ # Display a error and exit
+ fun error(msg: String) do
+ log.error "Error: {msg}"
+ exit 1
+ end
+end
+
+# Loader status by repo
+class LoaderJob
+ super RepoObject
+ serialize
+
+ # Repo this status is about
+ var repo: Repo
+
+ # Primary key: the repo id
+ redef var id is lazy, serialize_as("_id") do return repo.full_name
+
+ # Last issue loaded
+ var last_issue: Int
+end
+
+# Loader status repository
+class LoaderJobRepo
+ super MongoRepository[LoaderJob]
+end
+
+class RepoEntity
+ serialize
+
+ var repo: nullable Repo = null is writable
+end
+
+redef class Repo
+ serialize
+
+ var mongo_id: String is lazy, serialize_as("_id") do return full_name
+end
+
+class RepoRepo
+ super MongoRepository[Repo]
+end
+
+redef class Branch
+ super RepoEntity
+ serialize
+
+ var mongo_id: String is lazy, serialize_as("_id") do
+ var repo = self.repo
+ if repo == null then return name
+ return "{repo.mongo_id}/{name}"
+ end
+end
+
+class BranchRepo
+ super MongoRepository[Branch]
+
+ fun find_by_repo(repo: Repo): Array[Branch] do
+ return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
+ end
+end
+
+redef class Commit
+ super RepoEntity
+ serialize
+
+ var mongo_id: String is lazy, serialize_as("_id") do return sha
+end
+
+class CommitRepo
+ super MongoRepository[Commit]
+
+ fun find_by_repo(repo: Repo): Array[Commit] do
+ return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
+ end
+end
+
+redef class Issue
+ super RepoEntity
+ serialize
+
+ var mongo_id: String is lazy, serialize_as("_id") do
+ var repo = self.repo
+ if repo == null then return number.to_s
+ return "{repo.mongo_id}/{number}"
+ end
+end
+
+class IssueRepo
+ super MongoRepository[Issue]
+
+ fun find_by_repo(repo: Repo): Array[Issue] do
+ return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
+ end
+end
+
+class PullRequestRepo
+ super MongoRepository[PullRequest]
+
+ fun find_by_repo(repo: Repo): Array[Issue] do
+ return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
+ end
+end
+
+redef class IssueComment
+ super RepoEntity
+ serialize
+
+ var mongo_id: String is lazy, serialize_as("_id") do return id.to_s
+end
+
+class IssueCommentRepo
+ super MongoRepository[IssueComment]
+
+ fun find_by_repo(repo: Repo): Array[IssueComment] do
+ return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
+ end
+end
+
+redef class IssueEvent
+ super RepoEntity
+ serialize
+
+ var mongo_id: String is lazy, serialize_as("_id") do return id.to_s
+end
+
+class IssueEventRepo
+ super MongoRepository[IssueEvent]
+
+ fun find_by_repo(repo: Repo): Array[IssueEvent] do
+ return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
+ end
+end
+
+# Init options
+var loader = new Loader
+loader.config.parse_options(args)
+
+# TODO TMP
+loader.jobs.clear
+loader.repos.clear
+loader.branches.clear
+loader.commits.clear
+loader.issues.clear
+loader.pulls.clear
+loader.issue_comments.clear
+loader.issue_events.clear
+
+if loader.config.help then
+ loader.config.usage
+ exit 0
+end
+
+if loader.config.opt_show_wallet.value then
+ loader.show_wallet
+end
+
+var args = loader.config.args
+if loader.config.opt_show_jobs.value or args.is_empty then
+ loader.show_jobs
+end
+
+if args.is_empty then return
+
+if loader.config.opt_clear.value then
+ loader.remove args.first
+else
+ loader.start args.first
+
+ var repo = loader.config.wallet.api.load_repo(args.first)
+ if repo == null then return
+ print "Loaded"
+ print "* {if loader.repos.find_by_id(args.first) != null then 1 else 0} repos"
+ print "* {loader.branches.find_by_repo(repo).length} branches"
+ print "* {loader.commits.find_by_repo(repo).length} commits"
+ print "* {loader.issues.find_by_repo(repo).length} issues"
+ print "* {loader.pulls.find_by_repo(repo).length} pulls"
+ print "* {loader.issue_comments.find_by_repo(repo).length} comments"
+ print "* {loader.issue_events.find_by_repo(repo).length} events"
+end
# Lexer generated by nitcc for the grammar json
-module json_lexer is no_warning "missing-doc"
+module json_lexer is generated, no_warning "missing-doc"
import nitcc_runtime
import json_parser
class Lexer_json
# Parser generated by nitcc for the grammar json
-module json_parser is no_warning("missing-doc","old-init")
+module json_parser is generated, no_warning("missing-doc","old-init")
import nitcc_runtime
class Parser_json
super Parser
do
if not plain_json or not first_attribute then
stream.write ","
- first_attribute = false
end
+ first_attribute = false
new_line_and_indent
stream.write "\""
redef fun accept_json_serializer(v)
do
v.stream.write "\""
+
+ var start_i = 0
+ var escaped = null
for i in [0 .. self.length[ do
var char = self[i]
if char == '\\' then
- v.stream.write "\\\\"
+ escaped = "\\\\"
else if char == '\"' then
- v.stream.write "\\\""
+ escaped = "\\\""
else if char < ' ' then
if char == '\n' then
- v.stream.write "\\n"
+ escaped = "\\n"
else if char == '\r' then
- v.stream.write "\\r"
+ escaped = "\\r"
else if char == '\t' then
- v.stream.write "\\t"
+ escaped = "\\t"
else
- v.stream.write char.escape_to_utf16
+ escaped = char.escape_to_utf16
+ end
+ end
+
+ if escaped != null then
+ # Write open non-escaped string
+ if start_i <= i then
+ v.stream.write substring(start_i, i-start_i)
end
+
+ # Write escaped character
+ v.stream.write escaped
+ escaped = null
+ start_i = i+1
+ end
+ end
+
+ # Write remaining non-escaped string
+ if start_i < length then
+ if start_i == 0 then
+ v.stream.write self
else
- v.stream.write char.to_s
+ v.stream.write substring(start_i, length-start_i)
end
end
+
v.stream.write "\""
end
end
# Is there data are stored under `key`.
fun has_key(key: String): Bool do
- return (store_dir / "{key}.json").file_exists
+ return ("{store_dir}/{key}.json".simplify_path).file_exists
end
# Save `json` object under `key`.
# Only `JsonObject` and `JsonArray` are allowed in a json file.
# Use `store_object` or `store_array` instead.
private fun store_json(key: String, json: Jsonable) do
- var path = store_dir / "{key}.json"
+ var path = "{store_dir}/{key}.json".simplify_path
path.dirname.mkdir
var file = new FileWriter.open(path)
file.write(json.to_json)
# Ensure `has_data(key)`
private fun load_json(key: String): nullable Jsonable do
assert has_key(key)
- var path = store_dir / "{key}.json"
+ var path = "{store_dir}/{key}.json".simplify_path
var file = new FileReader.open(path)
var text = file.read_all
file.close
# Get the list of keys stored under the collection `key`.
fun list_collection(key: String): JsonArray do
var res = new JsonArray
- var coll = (store_dir / "{key}").to_path
+ var coll = ("{store_dir}/{key}".simplify_path).to_path
if not coll.exists or not coll.stat.is_dir then return res
for file in coll.files do
if file.to_s.has_suffix(".json") then
# Does `key` matches a collection?
fun has_collection(key: String): Bool do
- var path = (store_dir / "{key}").to_path
+ var path = ("{store_dir}/{key}".simplify_path).to_path
return path.exists and path.stat.is_dir
end
end
types["htm"] = "text/html"
types["shtml"] = "text/html"
types["css"] = "text/css"
+ types["csv"] = "text/csv"
types["xml"] = "text/xml"
types["rss"] = "text/xml"
types["gif"] = "image/gif"
types["ico"] = "image/x-icon"
types["jng"] = "image/x-jng"
types["wbmp"] = "image/vnd.wap.wbmp"
+ types["gz"] = "application/gzip"
types["jar"] = "application/java-archive"
types["war"] = "application/java-archive"
types["ear"] = "application/java-archive"
import pop_routes
import json::static
import json
+import csv
# Class handler for a route.
#
end
end
+ # Write data as CSV and set the right content type header.
+ fun csv(csv: nullable CsvDocument, status: nullable Int) do
+ header["Content-Type"] = media_types["csv"].as(not null)
+ if csv == null then
+ send(null, status)
+ else
+ send(csv.write_to_string, status)
+ end
+ end
+
# Write error as JSON and set the right content type header.
fun json_error(error: nullable Jsonable, status: nullable Int) do
json(error, status)
LDLIBS := $(filter-out -lrt,$(LDLIBS))
endif
+# Special configuration for Windows under mingw64
+ifeq ($(uname_S),MINGW64_NT-10.0)
+ # Use the pcreposix regex library
+ LDLIBS += -lpcreposix
+
+ # Remove POSIX flag -lrt
+ LDLIBS := $(filter-out -lrt,$(LDLIBS))
+
+ # Silence warnings when storing Int, Char and Bool as pointer address
+ CFLAGS += -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast
+endif
+
"""
makefile.write("all: {outpath}\n")
self.toolcontext.info(command, 2)
var res
- if self.toolcontext.verbose_level >= 3 or is_windows then
+ if self.toolcontext.verbose_level >= 3 then
res = sys.system("{command} 2>&1")
+ else if is_windows then
+ res = sys.system("{command} 2>&1 >nul")
else
res = sys.system("{command} 2>&1 >/dev/null")
end
intern
extern
no_warning
+generated
auto_inspect
end
# Is the module a test suite?
mmodule.is_test_suite = not decl.get_annotations("test_suite").is_empty
+ # Is the module generated?
+ mmodule.is_generated = not decl.get_annotations("generated").is_empty
end
end
# Is `self` a unit test module used by `nitunit`?
var is_test_suite: Bool = false is writable
+ # Is `self` a module generated by a tool?
+ #
+ # This flag has no effect on the semantic.
+ # It is only intended on software engineering software to discriminate computer-generated modules from human-written ones.
+ var is_generated: Bool = false is writable
+
# Get the non-`is_fictive` module on which `self` is based on.
#
# On non-fictive module, this returns `self`.
end
var nit_module = new NitModule(module_name)
+nit_module.annotations.add """generated"""
nit_module.annotations.add """no_warning("parentheses")"""
nit_module.header = """
# This file is generated by nitrestful
if importations == null then importations = target_modules
var nit_module = new NitModule(module_name)
+ nit_module.annotations.add """generated"""
nit_module.annotations.add """no_warning("property-conflict")"""
nit_module.header = """
# This file is generated by nitserial
# Lexer and its tokens.
# This file was generated by SableCC (http://www.sablecc.org/).
-module lexer is no_warning("missing-doc")
+module lexer is generated, no_warning("missing-doc")
intrude import parser_nodes
intrude import lexer_work
# Parser.
# This file was generated by SableCC (http://www.sablecc.org/).
-module parser is no_warning("missing-doc", "unread-variable")
+module parser is generated, no_warning("missing-doc", "unread-variable")
intrude import parser_prod
intrude import parser_work
# Raw AST node hierarchy.
# This file was generated by SableCC (http://www.sablecc.org/).
-module parser_abs is no_warning("missing-doc")
+module parser_abs is generated, no_warning("missing-doc")
import location
# Production AST nodes full definition.
# This file was generated by SableCC (http://www.sablecc.org/).
-module parser_prod is no_warning("missing-doc")
+module parser_prod is generated, no_warning("missing-doc")
import lexer
intrude import parser_nodes
$ output 'parser_abs.nit'
# Raw AST node hierarchy.
# This file was generated by SableCC (http://www.sablecc.org/).
-module parser_abs is no_warning("missing-doc")
+module parser_abs is generated, no_warning("missing-doc")
import location
$ output 'lexer.nit'
# Lexer and its tokens.
# This file was generated by SableCC (http://www.sablecc.org/).
-module lexer is no_warning("missing-doc")
+module lexer is generated, no_warning("missing-doc")
$ if $usermodule
intrude import $usermodule
$ output 'parser_prod.nit'
# Production AST nodes full definition.
# This file was generated by SableCC (http://www.sablecc.org/).
-module parser_prod is no_warning("missing-doc")
+module parser_prod is generated, no_warning("missing-doc")
import lexer
$ if $usermodule
$ output 'parser.nit'
# Parser.
# This file was generated by SableCC (http://www.sablecc.org/).
-module parser is no_warning("missing-doc", "unread-variable")
+module parser is generated, no_warning("missing-doc", "unread-variable")
intrude import parser_prod
intrude import parser_work
#
# It uses, in order:
#
- # * the option `opt_no_color`
+ # * the option `opt_nit_dir`
# * the environment variable `NIT_DIR`
# * the runpath of the program from argv[0]
# * the runpath of the process from /proc
end
# search in the PATH
- var ps = "PATH".environ.split(":")
+ var path_sep = if is_windows then ";" else ":"
+ var ps = "PATH".environ.split(path_sep)
for p in ps do
res = p/".."
if check_nit_dir(res) then return res.simplify_path
# This file is generated by nitrestful
# Do not modify, instead refine the generated services.
module restful_annot_rest is
+ generated
no_warning("parentheses")
end
# This file is generated by nitserial
# Do not modify, but you can redef
module test_serialization_serial is
+ generated
no_warning("property-conflict")
end
nitlanguage / nit.git / commitdiff