From 3a82fa187e484494e8f346c00c7172a68f22cad8 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Jean-Christophe=20Beaupr=C3=A9?= Date: Tue, 25 Nov 2014 11:51:11 -0500 Subject: [PATCH] =?utf8?q?neo=5Fdoxygen:=20Isolate=20Neo4j=E2=80=99s=20servi?= =?utf8?q?ces.?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit This will permit to test the program without requiring a Neo4j server. Signed-off-by: Jean-Christophe Beaupré --- contrib/neo_doxygen/src/graph_store.nit | 113 +++++++++++++++++++++++++++++++ contrib/neo_doxygen/src/neo_doxygen.nit | 80 +++++++--------------- 2 files changed, 138 insertions(+), 55 deletions(-) create mode 100644 contrib/neo_doxygen/src/graph_store.nit diff --git a/contrib/neo_doxygen/src/graph_store.nit b/contrib/neo_doxygen/src/graph_store.nit new file mode 100644 index 0000000..cc0eaec --- /dev/null +++ b/contrib/neo_doxygen/src/graph_store.nit @@ -0,0 +1,113 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A storage medium for a graph. +module graph_store + +import neo4j +import console + +# A storage medium for a graph. +# +# Provides a way to save a Neo4j graph. +abstract class GraphStore + + # Escape control sequence to save the cursor position. + private var term_save_cursor: String = (new TermSaveCursor).to_s + + # Escape control sequence to rewind to the last saved cursor position. + private var term_rewind: String = "{new TermRestoreCursor}{new TermEraseDisplayDown}" + + # Is the storage medium already contains at least one node with the specified label? + fun has_node_label(name: String): Bool is abstract + + # Save all specified Neo4j entities. + fun save_all(neo_entities: Collection[NeoEntity]) is abstract + + # Prepare the output to show the progress. + # + # This method must be called before the first call to `show_progress` or + # `show_done`. + protected fun prepare_display do + printn "{term_save_cursor} " + end + + # Show the progress, in percentage. + # + # For use in the implementation of `save_all` only. + protected fun show_progress(progress: Int) do + printn "{term_rewind} {progress}% " + end + + # Show a message to indicate that the task finished with success. + # + # For use in the implementation of `save_all` only. + protected fun show_done do + print "{term_rewind} Done." + end +end + +# An actual Neo4j database as a storage medium. +class Neo4jStore + super GraphStore + + # How many operations can be executed in one batch? + private var batch_max_size = 1000 + + # The Neo4j client to use. + var client: Neo4jClient + + redef fun has_node_label(name: String): Bool do + var query = new CypherQuery.from_string( + "match n where \{name\} in labels(n) return count(n)") + query.params["name"] = name + var data = client.cypher(query).as(JsonObject)["data"] + var result = data.as(JsonArray).first.as(JsonArray).first.as(Int) + return result > 0 + end + + redef fun save_all(neo_entities: Collection[NeoEntity]) do + var batch = new NeoBatch(client) + var len = neo_entities.length + var sum = 0 + var i = 1 + + prepare_display + for nentity in neo_entities do + batch.save_entity(nentity) + if i == batch_max_size then + do_batch(batch) + sum += batch_max_size + show_progress(sum * 100 / len) + batch = new NeoBatch(client) + i = 1 + else + i += 1 + end + end + do_batch(batch) + show_done + end + + # Execute `batch` and check for errors. + # + # Abort if `batch.execute` returns errors. + private fun do_batch(batch: NeoBatch) do + var errors = batch.execute + if not errors.is_empty then + for e in errors do sys.stderr.write("{sys.program_name}: {e}\n") + exit(1) + end + end +end diff --git a/contrib/neo_doxygen/src/neo_doxygen.nit b/contrib/neo_doxygen/src/neo_doxygen.nit index 7f38c01..3b160ad 100644 --- a/contrib/neo_doxygen/src/neo_doxygen.nit +++ b/contrib/neo_doxygen/src/neo_doxygen.nit @@ -20,21 +20,24 @@ module neo_doxygen import model import doxml +import graph_store import console import opts # An importation task. class NeoDoxygenJob - var client: Neo4jClient - var model: ProjectGraph is noinit - # How many operation can be executed in one batch? - private var batch_max_size = 1000 + # The storage medium to use. + var store: GraphStore + + # The loaded project graph. + var model: ProjectGraph is noinit - private var save_cursor: String = (new TermSaveCursor).to_s + # Escape control sequence to save the cursor position. + private var term_save_cursor: String = (new TermSaveCursor).to_s - # Escape control sequence to reset the current line. - private var reset_line: String = "{new TermRestoreCursor}{new TermEraseDisplayDown}" + # Escape control sequence to rewind to the last saved cursor position. + private var term_rewind: String = "{new TermRestoreCursor}{new TermEraseDisplayDown}" # Generate a graph from the specified project model. # @@ -52,9 +55,9 @@ class NeoDoxygenJob var file_count = 0 if dir == "" then - sys.stdout.write "Reading the current directory... " + printn "Reading the current directory... " else - sys.stdout.write "Reading {dir}... " + printn "Reading {dir}... " end loop for f in dir.files do @@ -83,11 +86,7 @@ class NeoDoxygenJob sys.stderr.write("{sys.program_name}: The project’s name must not" + " begin with an upper case letter. Got `{name}`.\n") end - var query = new CypherQuery.from_string("match n where \{name\} in labels(n) return count(n)") - query.params["name"] = name - var data = client.cypher(query).as(JsonObject)["data"] - var result = data.as(JsonArray).first.as(JsonArray).first.as(Int) - assert name_unused: result == 0 else + assert name_unused: not store.has_node_label(name) else sys.stderr.write("{sys.program_name}: The label `{name}` is already" + " used in the specified graph.\n") end @@ -95,49 +94,15 @@ class NeoDoxygenJob # Save the graph. fun save do - sys.stdout.write "Linking nodes...{save_cursor} " + sys.stdout.write "Linking nodes...{term_save_cursor} " model.put_edges - print "{reset_line} Done." + print "{term_rewind} Done." var nodes = model.all_nodes - sys.stdout.write "Saving {nodes.length} nodes...{save_cursor} " - push_all(nodes) + sys.stdout.write "Saving {nodes.length} nodes..." + store.save_all(nodes) var edges = model.all_edges - sys.stdout.write "Saving {edges.length} edges...{save_cursor} " - push_all(edges) - end - - # Save `neo_entities` in the database using batch mode. - private fun push_all(neo_entities: Collection[NeoEntity]) do - var batch = new NeoBatch(client) - var len = neo_entities.length - var sum = 0 - var i = 1 - - for nentity in neo_entities do - batch.save_entity(nentity) - if i == batch_max_size then - do_batch(batch) - sum += batch_max_size - sys.stdout.write("{reset_line} {sum * 100 / len}% ") - batch = new NeoBatch(client) - i = 1 - else - i += 1 - end - end - do_batch(batch) - print("{reset_line} Done.") - end - - # Execute `batch` and check for errors. - # - # Abort if `batch.execute` returns errors. - private fun do_batch(batch: NeoBatch) do - var errors = batch.execute - if not errors.is_empty then - for e in errors do sys.stderr.write("{sys.program_name}: {e}\n") - exit(1) - end + sys.stdout.write "Saving {edges.length} edges..." + store.save_all(edges) end end @@ -244,13 +209,18 @@ class NeoDoxygenCommand var dest = opt_dest.value var project_name = rest[0] var dir = rest[1] - var neo = new NeoDoxygenJob(new Neo4jClient(dest or else default_dest)) + var neo = new NeoDoxygenJob(create_store(dest or else default_dest)) neo.load_project(project_name, dir, source) neo.save return 0 end + # Create an instance of `GraphStore` for the specified destination. + protected fun create_store(dest: String): GraphStore do + return new Neo4jStore(new Neo4jClient(dest)) + end + # Show the help. fun show_help do option_context.usage -- 1.7.9.5