From: Jean-Christophe Beaupré <jcbrinfo@users.noreply.github.com>
Date: Mon, 3 Nov 2014 20:38:01 +0000 (-0500)
Subject: neo: Load the graph in chunks.
X-Git-Tag: v0.6.11~60^2~1
X-Git-Url: http://nitlanguage.org

neo: Load the graph in chunks.

Fix #870.

Signed-off-by: Jean-Christophe Beaupré <jcbrinfo@users.noreply.github.com>
---

diff --git a/lib/neo4j/curl_json.nit b/lib/neo4j/curl_json.nit
index f1a1da8..dc5c8ac 100644
--- a/lib/neo4j/curl_json.nit
+++ b/lib/neo4j/curl_json.nit
@@ -50,6 +50,7 @@ abstract class JsonCurlRequest
 		headers = new HeaderMap
 		headers["Accept"] = "application/json; charset=UTF-8"
 		headers["Transfer-Encoding"] = "chunked"
+		headers["X-Stream"] = "true"
 		if auth != null then
 			headers["Authorization"] = "token {auth.to_s}"
 		end
@@ -95,7 +96,7 @@ abstract class JsonCurlRequest
 		end
 
 		var err_hook = execute_hook
-	    if err_hook != null then return err_hook
+		if err_hook != null then return err_hook
 
 		var err_resp = perform
 		if err_resp != null then return err_resp
diff --git a/src/neo.nit b/src/neo.nit
index df2792c..d661481 100644
--- a/src/neo.nit
+++ b/src/neo.nit
@@ -213,27 +213,27 @@ class NeoModel
 
 	# Fill `model` using base pointed by `client`.
 	fun load(model: Model): Model do
-		toolcontext.info("Locate all mentities...", 1)
-		var nodes = client.nodes_with_label(model_name)
+		var nodes: Array[NeoNode]
 
-		toolcontext.info("Preload nodes...", 1)
-		pull_all_nodes(nodes)
-		toolcontext.info("Preload edges...", 1)
-		pull_all_edges(nodes)
-
-		toolcontext.info("Build model...", 1)
+		toolcontext.info("Loading project node...", 1)
 		nodes = client.nodes_with_labels([model_name, "MProject"])
 		for node in nodes do to_mproject(model, node)
+		toolcontext.info("Loading groups...", 1)
 		nodes = client.nodes_with_labels([model_name, "MGroup"])
 		for node in nodes do to_mgroup(model, node)
+		toolcontext.info("Loading modules...", 1)
 		nodes = client.nodes_with_labels([model_name, "MModule"])
 		for node in nodes do to_mmodule(model, node)
+		toolcontext.info("Loading classes...", 1)
 		nodes = client.nodes_with_labels([model_name, "MClass"])
 		for node in nodes do to_mclass(model, node)
+		toolcontext.info("Loading class definitions...", 1)
 		nodes = client.nodes_with_labels([model_name, "MClassDef"])
 		for node in nodes do to_mclassdef(model, node)
+		toolcontext.info("Loading properties...", 1)
 		nodes = client.nodes_with_labels([model_name, "MProperty"])
 		for node in nodes do to_mproperty(model, node)
+		toolcontext.info("Loading property definitions...", 1)
 		nodes = client.nodes_with_labels([model_name, "MPropDef"])
 		for node in nodes do to_mpropdef(model, node)
 		return model
@@ -270,52 +270,6 @@ class NeoModel
 		do_batch(batch)
 	end
 
-	# Load content for all `nodes` from base.
-	#
-	# Content corresponds to properties and labels that are loaded in batch mode.
-	private fun pull_all_nodes(nodes: Collection[NeoNode]) do
-		var batch = new NeoBatch(client)
-		var len = nodes.length
-		var sum = 0
-		var i = 1
-		for node in nodes do
-			batch.load_node(node)
-			if i == batch_max_size then
-				do_batch(batch)
-				sum += batch_max_size
-				toolcontext.info(" {sum * 100 / len}% done", 1)
-				batch = new NeoBatch(client)
-				i = 1
-			else
-				i += 1
-			end
-		end
-		do_batch(batch)
-	end
-
-	# Load all edges from base linked to `nodes`.
-	#
-	# Edges are loaded in batch mode.
-	private fun pull_all_edges(nodes: Collection[NeoNode]) do
-		var batch = new NeoBatch(client)
-		var len = nodes.length
-		var sum = 0
-		var i = 1
-		for node in nodes do
-			batch.load_node_edges(node)
-			if i == batch_max_size then
-				do_batch(batch)
-				sum += batch_max_size
-				toolcontext.info(" {sum * 100 / len}% done", 1)
-				batch = new NeoBatch(client)
-				i = 1
-			else
-				i += 1
-			end
-		end
-		do_batch(batch)
-	end
-
 	# How many operation can be executed in one batch?
 	private var batch_max_size = 1000