From 22a8c88c1ec794bb75cbc928cf0b509761ff18eb Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Fri, 16 Sep 2022 17:38:56 +0800 Subject: [PATCH 01/25] Fix bug: SchemaCache init and after query schema then put the latest schema to SchemaCache fix #333 --- .../java/com/baidu/hugegraph/loader/builder/SchemaCache.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java index 7253e0598..f709ee298 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java @@ -46,6 +46,7 @@ public SchemaCache(HugeClient client) { this.propertyKeys = new HashMap<>(); this.vertexLabels = new HashMap<>(); this.edgeLabels = new HashMap<>(); + updateAll(); } @JsonCreator @@ -90,6 +91,7 @@ public PropertyKey getPropertyKey(String name) { if (propertyKey == null) { try { propertyKey = this.client.schema().getPropertyKey(name); + updateAll(); } catch (ServerException e) { throw new LoadException("The property key '%s' doesn't exist", name); @@ -103,6 +105,7 @@ public VertexLabel getVertexLabel(String name) { if (vertexLabel == null) { try { vertexLabel = this.client.schema().getVertexLabel(name); + updateAll(); } catch (ServerException e) { throw new LoadException("The vertex label '%s' doesn't exist", name); @@ -116,6 +119,7 @@ public EdgeLabel getEdgeLabel(String name) { if (edgeLabel == null) { try { edgeLabel = this.client.schema().getEdgeLabel(name); + updateAll(); } catch (ServerException e) { throw new LoadException("The edge label '%s' doesn't exist", name); From d75f26321c9baba7aee3915616a32c5ea5ebe38a Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 20 Sep 2022 09:48:47 +0800 Subject: [PATCH 02/25] Fix bug: SchemaCache init and after query schema then put the latest schema to SchemaCache fix #333 --- .../java/com/baidu/hugegraph/loader/builder/SchemaCache.java | 4 ---- .../baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java index f709ee298..7253e0598 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java @@ -46,7 +46,6 @@ public SchemaCache(HugeClient client) { this.propertyKeys = new HashMap<>(); this.vertexLabels = new HashMap<>(); this.edgeLabels = new HashMap<>(); - updateAll(); } @JsonCreator @@ -91,7 +90,6 @@ public PropertyKey getPropertyKey(String name) { if (propertyKey == null) { try { propertyKey = this.client.schema().getPropertyKey(name); - updateAll(); } catch (ServerException e) { throw new LoadException("The property key '%s' doesn't exist", name); @@ -105,7 +103,6 @@ public VertexLabel getVertexLabel(String name) { if (vertexLabel == null) { try { vertexLabel = this.client.schema().getVertexLabel(name); - updateAll(); } catch (ServerException e) { throw new LoadException("The vertex label '%s' doesn't exist", name); @@ -119,7 +116,6 @@ public EdgeLabel getEdgeLabel(String name) { if (edgeLabel == null) { try { edgeLabel = this.client.schema().getEdgeLabel(name); - updateAll(); } catch (ServerException e) { throw new LoadException("The edge label '%s' doesn't exist", name); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java index ec94cf190..143dd2f0b 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java @@ -117,6 +117,7 @@ private LoadContext initPartition( this.builders.put(new EdgeBuilder(context, struct, edgeMapping), new ArrayList<>()); } + context.updateSchemaCache(); return context; } From d09a1f76f10ef4cce19179b4d14712e8c73a12ac Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 20 Sep 2022 13:48:53 +0800 Subject: [PATCH 03/25] Fix bug: flink-loader SchemaCache init fix #333 --- .../com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java index a37c73c37..edc503fde 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/flink/HugeGraphOutputFormat.java @@ -90,6 +90,7 @@ private Map> initBuilders() { builders.put(new EdgeBuilder(loadContext, this.struct, edgeMapping), new ArrayList<>()); } + loadContext.updateSchemaCache(); return builders; } From 9cef789166da3fd14a892e453d1a95be8e75ace9 Mon Sep 17 00:00:00 2001 From: alanzhao Date: Tue, 1 Nov 2022 00:30:40 +0800 Subject: [PATCH 04/25] =?UTF-8?q?bulkload=E4=BB=A3=E7=A0=81=E6=8F=90?= =?UTF-8?q?=E4=BA=A4=E7=A4=BE=E5=8C=BA=E7=AC=AC=E4=B8=80=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../assembly/static/bin/get-params.sh | 3 +- .../static/example/spark/edge_created.json | 4 + .../static/example/spark/edge_knows.json | 2 + .../static/example/spark/schema.groovy | 36 +++ .../assembly/static/example/spark/struct.json | 58 ++++ .../static/example/spark/vertex_person.json | 7 + .../static/example/spark/vertex_software.json | 3 + hugegraph-loader/pom.xml | 85 ++++++ .../hugegraph/loader/builder/EdgeBuilder.java | 50 +++- .../loader/builder/ElementBuilder.java | 2 + .../hugegraph/loader/builder/SchemaCache.java | 1 + .../loader/builder/VertexBuilder.java | 17 +- .../hugegraph/loader/constant/Constants.java | 4 + .../loader/direct/loader/DirectLoader.java | 69 +++++ .../direct/loader/HBaseDirectLoader.java | 271 ++++++++++++++++++ .../loader/direct/util/SinkToHBase.java | 158 ++++++++++ .../loader/executor/LoadOptions.java | 217 +++++++++----- .../loader/mapping/BackendStoreInfo.java | 76 +++++ .../hugegraph/loader/mapping/LoadMapping.java | 47 +-- .../loader/metrics/LoadDistributeMetrics.java | 184 ++++++++++++ .../loader/spark/HugeGraphSparkLoader.java | 218 ++++++++------ .../loader/struct/GraphStructV1.java | 8 + .../hugegraph/loader/util/MappingUtil.java | 6 +- 23 files changed, 1346 insertions(+), 180 deletions(-) create mode 100644 hugegraph-loader/assembly/static/example/spark/edge_created.json create mode 100644 hugegraph-loader/assembly/static/example/spark/edge_knows.json create mode 100644 hugegraph-loader/assembly/static/example/spark/schema.groovy create mode 100644 hugegraph-loader/assembly/static/example/spark/struct.json create mode 100644 hugegraph-loader/assembly/static/example/spark/vertex_person.json create mode 100644 hugegraph-loader/assembly/static/example/spark/vertex_software.json create mode 100644 hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java create mode 100644 hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java create mode 100644 hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java create mode 100644 hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java create mode 100644 hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java diff --git a/hugegraph-loader/assembly/static/bin/get-params.sh b/hugegraph-loader/assembly/static/bin/get-params.sh index c73ebbbd6..882e80874 100644 --- a/hugegraph-loader/assembly/static/bin/get-params.sh +++ b/hugegraph-loader/assembly/static/bin/get-params.sh @@ -27,7 +27,8 @@ function get_params() { --incremental-mode | --failure-mode | --batch-insert-threads | --single-insert-threads | \ --max-conn | --max-conn-per-route | --batch-size | --max-parse-errors | --max-insert-errors | \ --timeout | --shutdown-timeout | --retry-times | --retry-interval | --check-vertex | \ - --print-progress | --dry-run | --help) + --print-progress | --dry-run | --help | "--sink-type" | \ + "--vertex-partitions" | "--edge-partitions" |"--target-hdfs-conf") HUGEGRAPH_PARAMS="$HUGEGRAPH_PARAMS $1 $2" shift 2 ;; diff --git a/hugegraph-loader/assembly/static/example/spark/edge_created.json b/hugegraph-loader/assembly/static/example/spark/edge_created.json new file mode 100644 index 000000000..ba093eab1 --- /dev/null +++ b/hugegraph-loader/assembly/static/example/spark/edge_created.json @@ -0,0 +1,4 @@ +{"source_name": "marko", "target_id": 1, "date": "2017-12-10", "weight": 0.4} +{"source_name": "josh", "target_id": 1, "date": "2009-11-11", "weight": 0.4} +{"source_name": "josh", "target_id": 2, "date": "2017-12-10", "weight": 1.0} +{"source_name": "peter", "target_id": 1, "date": "2017-03-24", "weight": 0.2} diff --git a/hugegraph-loader/assembly/static/example/spark/edge_knows.json b/hugegraph-loader/assembly/static/example/spark/edge_knows.json new file mode 100644 index 000000000..0c2af9ad7 --- /dev/null +++ b/hugegraph-loader/assembly/static/example/spark/edge_knows.json @@ -0,0 +1,2 @@ +{"source_name": "marko", "target_name": "vadas", "date": "20160110", "weight": 0.5} +{"source_name": "marko", "target_name": "josh", "date": "20130220", "weight": 1.0} diff --git a/hugegraph-loader/assembly/static/example/spark/schema.groovy b/hugegraph-loader/assembly/static/example/spark/schema.groovy new file mode 100644 index 000000000..ea69bb530 --- /dev/null +++ b/hugegraph-loader/assembly/static/example/spark/schema.groovy @@ -0,0 +1,36 @@ +// Define schema +schema.propertyKey("name").asText().ifNotExist().create(); +schema.propertyKey("age").asInt().ifNotExist().create(); +schema.propertyKey("city").asText().ifNotExist().create(); +schema.propertyKey("weight").asDouble().ifNotExist().create(); +schema.propertyKey("lang").asText().ifNotExist().create(); +schema.propertyKey("date").asText().ifNotExist().create(); +schema.propertyKey("price").asDouble().ifNotExist().create(); + +schema.vertexLabel("person") + .properties("name", "age", "city") + .primaryKeys("name") + .nullableKeys("age", "city") + .ifNotExist() + .create(); +schema.vertexLabel("software") + .properties("name", "lang", "price") + .primaryKeys("name") + .ifNotExist() + .create(); + + +schema.edgeLabel("knows") + .sourceLabel("person") + .targetLabel("person") + .properties("date", "weight") + .ifNotExist() + .create(); +schema.edgeLabel("created") + .sourceLabel("person") + .targetLabel("software") + .properties("date", "weight") + .ifNotExist() + .create(); + + diff --git a/hugegraph-loader/assembly/static/example/spark/struct.json b/hugegraph-loader/assembly/static/example/spark/struct.json new file mode 100644 index 000000000..d5c7a94ab --- /dev/null +++ b/hugegraph-loader/assembly/static/example/spark/struct.json @@ -0,0 +1,58 @@ +{ + "vertices": [ + { + "label": "person", + "input": { + "type": "file", + "path": "/Users/alanzhao/Desktop/huge-jacky/incubator-hugegraph-toolchain/hugegraph-loader/assembly/static/example/spark/vertex_person.json", + "format": "JSON", + "header": ["name", "age", "city"], + "charset": "UTF-8", + "skipped_line": { + "regex": "(^#|^//).*" + } + }, + "id": "name", + "null_values": ["NULL", "null", ""] + }, + { + "label": "software", + "input": { + "type": "file", + "path": "/Users/alanzhao/Desktop/huge-jacky/incubator-hugegraph-toolchain/hugegraph-loader/assembly/static/example/spark/vertex_software.json", + "format": "JSON", + "header": ["id","name", "lang", "price","ISBN"], + "charset": "GBK" + }, + "id": "name", + "ignored": ["ISBN"] + } + ], + "edges": [ + { + "label": "knows", + "source": ["source_name"], + "target": ["target_name"], + "input": { + "type": "file", + "path": "/Users/alanzhao/Desktop/huge-jacky/incubator-hugegraph-toolchain/hugegraph-loader/assembly/static/example/spark/edge_knows.json", + "format": "JSON", + "date_format": "yyyyMMdd", + "header": ["source_name","target_name", "date", "weight"] + }, + "field_mapping": { + "source_name": "name", + "target_name": "name" + } + } + ], + "backendStoreInfo": + { + "edge_tablename": "hugegraph:g_oe", + "vertex_tablename": "hugegraph:g_v", + "hbase_zookeeper_quorum":"127.0.0.1", + "hbase_zookeeper_property_clientPort":"2181", + "zookeeper_znode_parent":"/hbase" + + } +} diff --git a/hugegraph-loader/assembly/static/example/spark/vertex_person.json b/hugegraph-loader/assembly/static/example/spark/vertex_person.json new file mode 100644 index 000000000..76fa1dd80 --- /dev/null +++ b/hugegraph-loader/assembly/static/example/spark/vertex_person.json @@ -0,0 +1,7 @@ +{"name": "marko", "age": "29", "city": "Beijing"} +{"name": "vadas", "age": "27", "city": "Hongkong"} +{"name": "josh", "age": "32", "city": "Beijing"} +{"name": "peter", "age": "35", "city": "Shanghai"} +{"name": "li,nary", "age": "26", "city": "Wu,han"} +{"name": "tom", "age": "null", "city": "NULL"} + diff --git a/hugegraph-loader/assembly/static/example/spark/vertex_software.json b/hugegraph-loader/assembly/static/example/spark/vertex_software.json new file mode 100644 index 000000000..ba68ffe8b --- /dev/null +++ b/hugegraph-loader/assembly/static/example/spark/vertex_software.json @@ -0,0 +1,3 @@ +{ "name": "lop", "lang": "java","price": "328","ISBN": "ISBN978-7-107-18618-5"} +{ "name": "ripple", "lang": "java","price": "199","ISBN": "ISBN978-7-100-13678-5"} + diff --git a/hugegraph-loader/pom.xml b/hugegraph-loader/pom.xml index 70132b082..6be79740f 100644 --- a/hugegraph-loader/pom.xml +++ b/hugegraph-loader/pom.xml @@ -337,6 +337,91 @@ + + org.apache.hbase + hbase-common + ${hbase.version} + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey + jersey-json + + + + + org.apache.hbase + hbase-client + ${hbase.version} + + + org.apache.hbase + hbase-shaded-client-byo-hadoop + + ${hbase.version} + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-ipc + + + + + org.apache.hbase + hbase-mapreduce + ${hbase.version} + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.apache.hadoop + hadoop-hdfs + + + + + org.apache.hugegraph + hugegraph-client + 1.0.0 + compile + diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java index 864e4f746..0b2c43ca1 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; +import com.baidu.hugegraph.loader.constant.Constants; import com.baidu.hugegraph.loader.executor.LoadContext; import com.baidu.hugegraph.loader.mapping.EdgeMapping; import com.baidu.hugegraph.loader.mapping.InputStruct; @@ -37,6 +38,7 @@ import com.baidu.hugegraph.structure.schema.VertexLabel; import com.baidu.hugegraph.util.E; import com.google.common.collect.ImmutableList; +import org.apache.spark.sql.Row; public class EdgeBuilder extends ElementBuilder { @@ -68,7 +70,6 @@ public EdgeBuilder(LoadContext context, InputStruct struct, public EdgeMapping mapping() { return this.mapping; } - @Override public List build(String[] names, Object[] values) { if (this.vertexIdsIndex == null || @@ -110,6 +111,53 @@ public List build(String[] names, Object[] values) { return edges; } + @Override + public List build( Row row) { + String[] names = row.schema().fieldNames(); + Object[] values = new Object[row.size()]; + for (int i = 0; i < row.size(); i++) { + values[i] = row.get(i); + } + if (this.vertexIdsIndex == null || + !Arrays.equals(this.lastNames, names)) { + this.vertexIdsIndex = this.extractVertexIdsIndex(names); + } + + this.lastNames = names; + EdgeKVPairs kvPairs = this.newEdgeKVPairs(); + kvPairs.source.extractFromEdge(names, values, + this.vertexIdsIndex.sourceIndexes); + kvPairs.target.extractFromEdge(names, values, + this.vertexIdsIndex.targetIndexes); + kvPairs.extractProperties(names, values); + + List sources = kvPairs.source.buildVertices(false); + List targets = kvPairs.target.buildVertices(false); + if (sources.isEmpty() || targets.isEmpty()) { + return ImmutableList.of(); + } + E.checkArgument(sources.size() == 1 || targets.size() == 1 || + sources.size() == targets.size(), + "The elements number of source and target must be: " + + "1 to n, n to 1, n to n"); + int size = Math.max(sources.size(), targets.size()); + List edges = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + Vertex source = i < sources.size() ? + sources.get(i) : sources.get(0); + Vertex target = i < targets.size() ? + targets.get(i) : targets.get(0); + Edge edge = new Edge(this.mapping.label()); + edge.source(source); + edge.target(target); + // Add properties + this.addProperties(edge, kvPairs.properties); + this.checkNonNullableKeys(edge); + edges.add(edge); + } + return edges; + } + private EdgeKVPairs newEdgeKVPairs() { EdgeKVPairs kvPairs = new EdgeKVPairs(); kvPairs.source = this.newKVPairs(this.sourceLabel, diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/ElementBuilder.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/ElementBuilder.java index fa280c6ec..176871366 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/ElementBuilder.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/ElementBuilder.java @@ -54,6 +54,7 @@ import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.LongEncoding; import com.google.common.collect.ImmutableList; +import org.apache.spark.sql.Row; public abstract class ElementBuilder { @@ -74,6 +75,7 @@ public ElementBuilder(LoadContext context, InputStruct struct) { public abstract ElementMapping mapping(); public abstract List build(String[] names, Object[] values); + public abstract List build( Row row); public abstract SchemaLabel schemaLabel(); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java index 7253e0598..e02460540 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java @@ -46,6 +46,7 @@ public SchemaCache(HugeClient client) { this.propertyKeys = new HashMap<>(); this.vertexLabels = new HashMap<>(); this.edgeLabels = new HashMap<>(); + updateAll(); } @JsonCreator diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java index f6bbb2f5f..64e3f26cf 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java @@ -29,6 +29,7 @@ import com.baidu.hugegraph.structure.schema.SchemaLabel; import com.baidu.hugegraph.structure.schema.VertexLabel; import com.baidu.hugegraph.util.E; +import org.apache.spark.sql.Row; public class VertexBuilder extends ElementBuilder { @@ -50,7 +51,6 @@ public VertexBuilder(LoadContext context, InputStruct struct, public VertexMapping mapping() { return this.mapping; } - @Override public List build(String[] names, Object[] values) { VertexKVPairs kvPairs = this.newKVPairs(this.vertexLabel, @@ -59,6 +59,21 @@ public List build(String[] names, Object[] values) { return kvPairs.buildVertices(true); } + @Override + public List build(Row row) { + + VertexKVPairs kvPairs = this.newKVPairs(this.vertexLabel, + this.mapping.unfold()); + String[] names = row.schema().fieldNames(); + Object[] values = new Object[row.size()]; + for (int i = 0; i < row.size(); i++) { + values[i] = row.get(i); + } + kvPairs.extractFromVertex(names, values); + return kvPairs.buildVertices(true); + + } + @Override public SchemaLabel schemaLabel() { return this.vertexLabel; diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java index 6f577f00c..cf8f0852f 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java @@ -79,4 +79,8 @@ public final class Constants { public static final String CDC_DATA = "data"; public static final String CDC_OP = "op"; + public static final String HBASE_COL_FAMILY="f"; + public static final String LOAD_DATA_PARSE_SUFFIX = "parse"; + public static final String LOAD_DATA_SER_SUFFIX = "ser"; + public static final String LOAD_DATA_INSERT_SUFFIX = "insert"; } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java new file mode 100644 index 000000000..b66bed7da --- /dev/null +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java @@ -0,0 +1,69 @@ +package com.baidu.hugegraph.loader.direct.loader; + +import com.baidu.hugegraph.loader.builder.EdgeBuilder; +import com.baidu.hugegraph.loader.builder.ElementBuilder; +import com.baidu.hugegraph.loader.builder.VertexBuilder; +import com.baidu.hugegraph.loader.executor.LoadContext; +import com.baidu.hugegraph.loader.executor.LoadOptions; +import com.baidu.hugegraph.loader.mapping.EdgeMapping; +import com.baidu.hugegraph.loader.mapping.InputStruct; +import com.baidu.hugegraph.loader.mapping.VertexMapping; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; + +import java.io.Serializable; +import java.util.LinkedList; +import java.util.List; + +/** + * 1.build + * 2.ser + * 3.generateFile + * 4.load + */ +public abstract class DirectLoader implements Serializable { + LoadOptions loadOptions ; + InputStruct struct; + + + public DirectLoader(LoadOptions loadOptions, + InputStruct struct) { + this.loadOptions = loadOptions; + this.struct=struct; + } + + + + + public void bulkload(Dataset ds){ + JavaPairRDD javaPairRDD = buildVertexAndEdge(ds); + String path = generateFiles(javaPairRDD); + loadFiles(path); + }; + + + protected List getElementBuilders(){ + LoadContext context = new LoadContext(loadOptions); + List buildersForGraphElement = new LinkedList<>(); + for (VertexMapping vertexMapping : struct.vertices()) { + buildersForGraphElement.add( + new VertexBuilder(context, struct, vertexMapping) + ); + } + for (EdgeMapping edgeMapping : struct.edges()) { + buildersForGraphElement.add(new EdgeBuilder(context, struct, edgeMapping)); + } + context.close(); + return buildersForGraphElement; + } + + + abstract JavaPairRDD buildVertexAndEdge(Dataset ds); + + + + abstract String generateFiles(JavaPairRDD buildAndSerRdd); + + abstract void loadFiles(String path); +} diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java new file mode 100644 index 000000000..fe0f57a47 --- /dev/null +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java @@ -0,0 +1,271 @@ +package com.baidu.hugegraph.loader.direct.loader; + +import com.baidu.hugegraph.loader.builder.ElementBuilder; +import com.baidu.hugegraph.loader.constant.Constants; + +import com.baidu.hugegraph.loader.direct.util.SinkToHBase; +import com.baidu.hugegraph.loader.executor.LoadOptions; +import com.baidu.hugegraph.loader.mapping.ElementMapping; +import com.baidu.hugegraph.loader.mapping.InputStruct; +import com.baidu.hugegraph.loader.metrics.LoadDistributeMetrics; +import com.baidu.hugegraph.loader.source.file.FileSource; +import com.baidu.hugegraph.loader.util.HugeClientHolder; +import com.baidu.hugegraph.loader.util.LoadUtil; +import com.baidu.hugegraph.serializer.direct.HBaseSerializer; +import com.baidu.hugegraph.structure.GraphElement; +import com.baidu.hugegraph.structure.graph.Edge; +import com.baidu.hugegraph.structure.graph.Vertex; +import com.baidu.hugegraph.util.Log; +import com.baidu.hugegraph.util.StringUtil; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsShell; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.spark.Partitioner; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.function.PairFlatMapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.slf4j.Logger; +import scala.Tuple2; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +public class HBaseDirectLoader extends DirectLoader { + + private SinkToHBase sinkToHBase ; + private LoadDistributeMetrics loadDistributeMetrics;; + + public static final Logger LOG = Log.logger(HBaseDirectLoader.class); + + public HBaseDirectLoader(LoadOptions loadOptions, InputStruct struct, LoadDistributeMetrics loadDistributeMetrics) { + super(loadOptions,struct); + this.loadDistributeMetrics=loadDistributeMetrics; + this.sinkToHBase=new SinkToHBase(loadOptions); + + } + + public String getTableName(){ + + String tableName = null; + if (struct.edges().size() > 0) {//导入的是边 表名换成 出边表 + System.out.println("this is edge struct: " + struct.edges().size()); + tableName = this.loadOptions.edgeTablename; + + } else if (struct.vertices().size() > 0) { + System.out.println("this is vertices struct: " + struct.vertices().size()); + tableName = this.loadOptions.vertexTablename; + + } + return tableName; + } + + public Integer getTablePartitions(){ + return struct.edges().size() > 0 ? loadOptions.edgePartitions + : loadOptions.vertexPartitions; + } + + public JavaPairRDD buildVertexAndEdge(Dataset ds) { + LOG.info("buildAndSer start execute >>>>"); + JavaPairRDD tuple2KeyValueJavaPairRDD = ds.toJavaRDD().mapPartitionsToPair( + new PairFlatMapFunction, ImmutableBytesWritable, KeyValue>() { + + @Override + public Iterator> call(Iterator rowIterator) throws Exception { + + List elementsElement;//存储返回的 GraphElement: Vertex/Edge + + // 目标存储序列化器 + HBaseSerializer serializer = new HBaseSerializer(HugeClientHolder.create(loadOptions),loadOptions.vertexPartitions,loadOptions.edgePartitions); + List buildersForGraphElement = getElementBuilders(); + + /** + * 构建 vertex/edge ---> K-V + */ + List> result = new LinkedList<>(); + while (rowIterator.hasNext()) { + Row row = rowIterator.next(); + List> serList = buildAndSer(serializer, row,buildersForGraphElement); + result.addAll(serList); + } + serializer.close(); + return result.iterator(); + } + } + ); + return tuple2KeyValueJavaPairRDD; + } + + + @Override + String generateFiles(JavaPairRDD buildAndSerRdd) { + LOG.info("bulkload start execute>>>"); + try { + Tuple2 tuple = + sinkToHBase.getPartitionerByTableName(getTablePartitions(), getTableName()); + Partitioner partitioner= (Partitioner) tuple._1; + TableDescriptor tableDescriptor= (TableDescriptor) tuple._2; + + JavaPairRDD repartitionedRdd = + buildAndSerRdd.repartitionAndSortWithinPartitions(partitioner);//精髓 partition内部做到有序 + Configuration conf = sinkToHBase.getHBaseConfiguration().get(); + Job job = Job.getInstance(conf); + LOG.info("hbasedirectloader HFileOutputFormat2 tableDescriptor >>>"+tableDescriptor.toString()); + HFileOutputFormat2.configureIncrementalLoadMap(job, tableDescriptor); + conf.set("hbase.mapreduce.hfileoutputformat.table.name", tableDescriptor.getTableName().getNameAsString()); + String path= getHFilePath(job.getConfiguration()); + + /** + * 准备bulkload相关配置信息 + */ + /** + * 生成 HFile + */ + LOG.info("hbasedirectloader HFileOutputFormat2 tablename >>>"+conf.get("hbase.mapreduce.hfileoutputformat.table.name")); + repartitionedRdd.saveAsNewAPIHadoopFile( + path, + ImmutableBytesWritable.class, + KeyValue.class, + HFileOutputFormat2.class, + conf + ); + LOG.info("Saved to HFiles to: " + path); + flushPermission(conf,path); + return path; + } catch (IOException e) { + e.printStackTrace(); + } + + return Constants.EMPTY_STR; + + } + + public String getHFilePath(Configuration conf) throws IOException { + FileSystem fs = FileSystem.get(conf); + long timeStr = System.currentTimeMillis(); + String pathStr = fs.getWorkingDirectory().toString() + "/hfile-gen" + "/" + timeStr+ "/";//HFile 存储路径 + Path hfileGenPath = new Path(pathStr); + if(fs.exists(hfileGenPath)){ + LOG.info("\n delete hfile path \n"); + fs.delete(hfileGenPath,true); + } + return pathStr; + } + + + @Override + public void loadFiles(String path) { + long timeStr = System.currentTimeMillis(); + + try { + sinkToHBase.loadHfiles(path, getTableName());// BulkLoad HFile to HBase + } catch (Exception e) { + e.printStackTrace(); + } + + + } + + private void flushPermission(Configuration conf, String path){ + FsShell shell = new FsShell(conf); + try { + LOG.info("shell start execute"); + shell.run(new String[]{"-chmod", "-R", "777", path}); + shell.close(); + } catch (Exception e) { + LOG.error("Couldnt change the file permissions " + e + + " Please run command:" + + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles " + + path + " '" + + "test" + "'\n" + + " to load generated HFiles into HBase table"); + } + } + + // 构建点边然后序列化为目标存储的所需类型 + List> buildAndSer(HBaseSerializer serializer, Row row, List buildersForGraphElement) { + List elementsElement;//存储返回的 GraphElement: Vertex/Edge + + List> result = new LinkedList<>(); + + for (ElementBuilder builder : buildersForGraphElement) { // 控制类型 点/边 + ElementMapping elementMapping = builder.mapping(); + if (elementMapping.skip()) { + continue; + } + if ("".equals(row.mkString())) { + break; + } + switch (struct.input().type()) { + case FILE: + case HDFS: + + elementsElement = builder.build(row); + + break; + default: + throw new AssertionError(String.format( + "Unsupported input source '%s'", + struct.input().type())); + } + + boolean isVertex = builder.mapping().type().isVertex(); + if (isVertex) { + for (Vertex vertex : (List) (Object) elementsElement) { + LOG.info("vertex already build done >>>" + vertex.toString()); + Tuple2 tuple2 = vertexSerialize(serializer,vertex); + loadDistributeMetrics.increaseDisVertexInsertSuccess(builder.mapping()); + result.add(tuple2); + } + } else { + for (Edge edge : (List) (Object) elementsElement) { + LOG.info("edge already build done >>>" + edge.toString()); + Tuple2 tuple2 = edgeSerialize(serializer,edge); + loadDistributeMetrics.increaseDisEdgeInsertSuccess(builder.mapping()); + result.add(tuple2); + + } + } + } + return result; + } + + private Tuple2 edgeSerialize(HBaseSerializer serializer, Edge edge) { + LOG.info("edge start serialize >>>" + edge.toString()); + byte[] rowkey = serializer.getKeyBytes(edge); + byte[] values = serializer.getValueBytes(edge); + ImmutableBytesWritable rowKey = new ImmutableBytesWritable(); + rowKey.set(rowkey); + KeyValue keyValue = new KeyValue(rowkey, + Bytes.toBytes(Constants.HBASE_COL_FAMILY), + Bytes.toBytes(Constants.EMPTY_STR), + values); + return new Tuple2<>(rowKey,keyValue); + } + + private Tuple2 vertexSerialize(HBaseSerializer serializer, Vertex vertex) { + LOG.info("vertex start serialize >>>" + vertex.toString()); + byte[] rowkey = serializer.getKeyBytes(vertex); + byte[] values = serializer.getValueBytes(vertex); + ImmutableBytesWritable rowKey = new ImmutableBytesWritable(); + rowKey.set(rowkey); + KeyValue keyValue = new KeyValue(rowkey, + Bytes.toBytes(Constants.HBASE_COL_FAMILY), + Bytes.toBytes(Constants.EMPTY_STR), + values); + return new Tuple2<>(rowKey,keyValue); + } + + +} diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java new file mode 100644 index 000000000..159a66019 --- /dev/null +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java @@ -0,0 +1,158 @@ +package com.baidu.hugegraph.loader.direct.util; + +import com.baidu.hugegraph.loader.executor.LoadOptions; +import com.baidu.hugegraph.util.Log; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.spark.Partitioner; +import org.slf4j.Logger; +import scala.Tuple2; + +import java.io.IOException; +import java.io.Serializable; +import java.util.*; + + +public class SinkToHBase implements Serializable { + private LoadOptions loadOptions; + public static final Logger LOG = Log.logger(SinkToHBase.class); + + public SinkToHBase(LoadOptions loadOptions) { + this.loadOptions=loadOptions; + } + + + public Optional getHBaseConfiguration(){ + Configuration baseConf = HBaseConfiguration.create(); + baseConf.set("hbase.zookeeper.quorum", this.loadOptions.hbaseZKQuorum); + baseConf.set("hbase.zookeeper.property.clientPort", this.loadOptions.hbaseZKPort); + baseConf.set("zookeeper.znode.parent", this.loadOptions.hbaseZKParent); + + return Optional.ofNullable(baseConf); + } + + private Optional getConnection(){ + Optional baseConf = getHBaseConfiguration(); + Connection conn=null; + try { + conn = ConnectionFactory.createConnection(baseConf.get()); + } catch (IOException e) { + e.printStackTrace(); + } + return Optional.ofNullable(conn); + } + public Tuple2 getPartitionerByTableName (int numPartitions, String tableName) throws IOException { + LOG.debug("getPartitionerByTableName 接收的tablename 》》》"+tableName); + Optional optionalConnection = getConnection(); + TableDescriptor descriptor = optionalConnection + .get() + .getTable(TableName.valueOf(tableName)) + .getDescriptor(); + LOG.debug("getPartitionerByTableName get TableDescriptor 》》》"+descriptor.getTableName()); + optionalConnection.get().close(); + return new Tuple2(new IntPartitioner(numPartitions, tableName),descriptor); + } + + public void loadHfiles (String path, String tableName) throws Exception { + Connection conn = getConnection().get(); + + Table table = conn.getTable(TableName.valueOf(tableName)); + Configuration conf = conn.getConfiguration(); + System.out.println("conf >>"+conf.get("dfs.client.failover.proxy.provider.hwyriskmgtcluster")); + System.out.println("conf >>"+conf.get("dfs.ha.namenodes.hwyriskmgtcluster")); + System.out.println("conf >>"+conf.get("dfs.namenode.http-address.hwyriskmgtcluster.nn1")); + System.out.println("conf >>"+conf.get("dfs.namenode.http-address.hwyriskmgtcluster.nn2")); + BulkLoadHFilesTool bulkLoadHFilesTool = new BulkLoadHFilesTool(conf); + + //TODO: load HFile to HBase + bulkLoadHFilesTool.bulkLoad(table.getName(), new Path(path)); + table.close(); + conn.close(); + + } + + + //精髓 repartitionAndSort + public class IntPartitioner extends Partitioner { + private final int numPartitions; + public Map, Integer> rangeMap = new HashMap<>(); + private String tableName; + + public IntPartitioner(int numPartitions, String tableName) throws IOException { + this.numPartitions = numPartitions; + this.rangeMap = getRangeMap(tableName); + this.tableName = tableName; + } + + + private Map, Integer> getRangeMap(String tableName) throws IOException { + Connection conn = getConnection().get(); + + HRegionLocator locator = (HRegionLocator) conn.getRegionLocator(TableName.valueOf(tableName)); + + Pair startEndKeys = locator.getStartEndKeys(); + + Map, Integer> rangeMap = new HashMap<>(); + for (int i = 0; i < startEndKeys.getFirst().length; i++) { + String startKey = Bytes.toString(startEndKeys.getFirst()[i]); + String endKey = Bytes.toString(startEndKeys.getSecond()[i]); + + rangeMap.put(new ArrayList<>(Arrays.asList(startKey, endKey)), i); + } + conn.close(); + return rangeMap; + } + + @Override + public int numPartitions() { + return numPartitions; + } + + @Override + public int getPartition(Object key) { + if (key instanceof ImmutableBytesWritable) { + + try { + ImmutableBytesWritable immutableBytesWritableKey = (ImmutableBytesWritable) key; + + if (rangeMap == null || rangeMap.isEmpty()) { + rangeMap = getRangeMap(this.tableName); + } + + String keyString = Bytes.toString(immutableBytesWritableKey.get()); + for (List range : rangeMap.keySet()) { + if (keyString.compareToIgnoreCase(range.get(0)) >= 0 + && ((keyString.compareToIgnoreCase(range.get(1)) < 0) + || range.get(1).equals(""))) { + return rangeMap.get(range); + } + } + LOG.error("Didn't find proper key in rangeMap, so returning 0 ..."); + return 0; + } catch (Exception e) { + LOG.error("When trying to get partitionID, " + + "encountered exception: " + e + "\t key = " + key); + return 0; + } + } else { + LOG.error("key is NOT ImmutableBytesWritable type ..."); + return 0; + } + } + } + +} + + + + + + + diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java index ac12d53b7..5934b34ae 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java @@ -23,6 +23,7 @@ import java.io.Serializable; import java.util.Set; +import com.baidu.hugegraph.loader.mapping.BackendStoreInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -44,168 +45,202 @@ public class LoadOptions implements Serializable { public static final String HTTP_SCHEMA = "http"; private static final int CPUS = Runtime.getRuntime().availableProcessors(); + @Parameter(names = {"-f", "--file"}, required = true, arity = 1, - validateWith = {FileValidator.class}, - description = "The path of the data mapping description file") + validateWith = {FileValidator.class}, + description = "The path of the data mapping description file") public String file; @Parameter(names = {"-s", "--schema"}, arity = 1, - validateWith = {FileValidator.class}, - description = "The schema file path which to create manually") + validateWith = {FileValidator.class}, + description = "The schema file path which to create manually") public String schema; @Parameter(names = {"-g", "--graph"}, required = true, arity = 1, - description = "The namespace of the graph to load into") + description = "The namespace of the graph to load into") public String graph; @Parameter(names = {"-h", "--host"}, arity = 1, - validateWith = {UrlValidator.class}, - description = "The host/IP of HugeGraphServer") + validateWith = {UrlValidator.class}, + description = "The host/IP of HugeGraphServer") public String host = "localhost"; @Parameter(names = {"-p", "--port"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The port of HugeGraphServer") + validateWith = {PositiveValidator.class}, + description = "The port of HugeGraphServer") public int port = 8080; @Parameter(names = {"--username"}, arity = 1, - description = "The username of graph for authentication") + description = "The username of graph for authentication") public String username = null; @Parameter(names = {"--protocol"}, arity = 1, - validateWith = {ProtocolValidator.class}, - description = "The protocol of HugeGraphServer, " + - "allowed values are: http or https") + validateWith = {ProtocolValidator.class}, + description = "The protocol of HugeGraphServer, " + + "allowed values are: http or https") public String protocol = "http"; @Parameter(names = {"--trust-store-file"}, arity = 1, - description = "The path of client truststore file used " + - "when https protocol is enabled") + description = "The path of client truststore file used " + + "when https protocol is enabled") public String trustStoreFile = null; @Parameter(names = {"--trust-store-password"}, arity = 1, - description = "The password of client truststore file used " + - "when https protocol is enabled") + description = "The password of client truststore file used " + + "when https protocol is enabled") public String trustStoreToken = null; @Parameter(names = {"--token"}, arity = 1, - description = "The token of graph for authentication") + description = "The token of graph for authentication") public String token = null; @Parameter(names = {"--clear-all-data"}, arity = 1, - description = "Whether to clear all old data before loading") + description = "Whether to clear all old data before loading") public boolean clearAllData = false; @Parameter(names = {"--clear-timeout"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The timeout waiting for clearing all data ") + validateWith = {PositiveValidator.class}, + description = "The timeout waiting for clearing all data ") public int clearTimeout = 240; @Parameter(names = {"--incremental-mode"}, arity = 1, - description = "Load data from the breakpoint of last time") + description = "Load data from the breakpoint of last time") public boolean incrementalMode = false; @Parameter(names = {"--failure-mode"}, arity = 1, - description = "Load data from the failure records, in this " + - "mode, only full load is supported, any read " + - "or parsing errors will cause load task stop") + description = "Load data from the failure records, in this " + + "mode, only full load is supported, any read " + + "or parsing errors will cause load task stop") public boolean failureMode = false; @Parameter(names = {"--batch-insert-threads"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The number of threads to execute batch insert") + validateWith = {PositiveValidator.class}, + description = "The number of threads to execute batch insert") public int batchInsertThreads = CPUS; @Parameter(names = {"--single-insert-threads"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The number of threads to execute single insert") + validateWith = {PositiveValidator.class}, + description = "The number of threads to execute single insert") public int singleInsertThreads = 8; @Parameter(names = {"--max-conn"}, arity = 1, - description = "Max number of HTTP connections to server") + description = "Max number of HTTP connections to server") public int maxConnections = CPUS * 4; @Parameter(names = {"--max-conn-per-route"}, arity = 1, - description = "Max number of HTTP connections to each route") + description = "Max number of HTTP connections to each route") public int maxConnectionsPerRoute = CPUS * 2; @Parameter(names = {"--batch-size"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The number of lines in each submit") + validateWith = {PositiveValidator.class}, + description = "The number of lines in each submit") public int batchSize = 500; @Parameter(names = {"--cdc-flush-interval"}, arity = 1, - description = "The flush interval for flink cdc") + description = "The flush interval for flink cdc") public int flushIntervalMs = 30000; @Parameter(names = {"--cdc-sink-parallelism"}, arity = 1, - description = "The sink parallelism for flink cdc") + description = "The sink parallelism for flink cdc") public int sinkParallelism = 1; @Parameter(names = {"--shutdown-timeout"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The timeout of awaitTermination in seconds") + validateWith = {PositiveValidator.class}, + description = "The timeout of awaitTermination in seconds") public int shutdownTimeout = 10; @Parameter(names = {"--check-vertex"}, arity = 1, - description = "Check vertices exists while inserting edges") + description = "Check vertices exists while inserting edges") public boolean checkVertex = false; @Parameter(names = {"--max-read-errors"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The maximum number of lines that read error " + - "before exiting") + validateWith = {PositiveValidator.class}, + description = "The maximum number of lines that read error " + + "before exiting") public int maxReadErrors = 1; @Parameter(names = {"--max-parse-errors"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The maximum number of lines that parse error " + - "before exiting") + validateWith = {PositiveValidator.class}, + description = "The maximum number of lines that parse error " + + "before exiting") public int maxParseErrors = 1; @Parameter(names = {"--max-insert-errors"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The maximum number of lines that insert error " + - "before exiting") + validateWith = {PositiveValidator.class}, + description = "The maximum number of lines that insert error " + + "before exiting") public int maxInsertErrors = 500; @Parameter(names = {"--timeout"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The timeout of HugeClient request") + validateWith = {PositiveValidator.class}, + description = "The timeout of HugeClient request") public int timeout = 60; @Parameter(names = {"--retry-times"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "Setting the max retry times when loading timeout") + validateWith = {PositiveValidator.class}, + description = "Setting the max retry times when loading timeout") public int retryTimes = 3; @Parameter(names = {"--retry-interval"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "Setting the interval time before retrying") + validateWith = {PositiveValidator.class}, + description = "Setting the interval time before retrying") public int retryInterval = 10; @Parameter(names = {"--max-read-lines"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The maximum number of read lines, when reached " + - "this number, the load task will stop") + validateWith = {PositiveValidator.class}, + description = "The maximum number of read lines, when reached " + + "this number, the load task will stop") public long maxReadLines = -1L; @Parameter(names = {"--dry-run"}, arity = 1, - description = "Dry run means that only parse but doesn't load") + description = "Dry run means that only parse but doesn't load") public boolean dryRun = false; @Parameter(names = {"--print-progress"}, arity = 1, - description = "Whether to print real-time load progress") + description = "Whether to print real-time load progress") public boolean printProgress = true; @Parameter(names = {"--test-mode"}, arity = 1, - description = "Whether the hugegraph-loader work in test mode") + description = "Whether the hugegraph-loader work in test mode") public boolean testMode = false; @Parameter(names = {"--help"}, help = true, - description = "Print usage of HugeGraphLoader") + description = "Print usage of HugeGraphLoader") public boolean help; + @Parameter(names = {"--sink-type"}, arity = 1, + description = "Sink to different storage") + public boolean sinkType = true; + + + @Parameter(names = {"--edge-partitions"}, arity = 1, + description = "The number of partitions of the HBase edge table") + public int edgePartitions = 64; + + @Parameter(names = {"--vertex-partitions"}, arity = 1, + description = "The number of partitions of the HBase vertex table") + public int vertexPartitions = 64; + + @Parameter(names = {"edgeTablename"}, arity = 1, + description = "edgeTablename") + public String edgeTablename; + @Parameter(names = {"vertexTablename"}, arity = 1, + description = "vertexTablename") + public String vertexTablename; + @Parameter(names = {"hbaseZKQuorum"}, arity = 1, + description = "hbaseZKQuorum") + public String hbaseZKQuorum; + @Parameter(names = {"hbaseZKPort"}, arity = 1, + description = "hbaseZKPort") + public String hbaseZKPort; + @Parameter(names = {"hbaseZKParent"}, arity = 1, + description = "hbaseZKParent") + public String hbaseZKParent; + @Parameter(names = {"targetHdfsHfilePath"}, arity = 1, + description = "target hdfs hfile path,need to execute distcp ") + public String targetHdfsHfilePath; + + public String workModeString() { if (this.incrementalMode) { return "INCREMENTAL MODE"; @@ -216,11 +251,24 @@ public String workModeString() { } } + /** + * --username admin + * --token admin + * -g hugegraph + * -f /home/data/wwwuser/social-graph-struct.json + * -s /home/data/wwwuser/hugegraph-loader/hugegraph-loader-1.0.0 + * /example/social-graph/social-graph-schema.groovy + * --batch-size 500 + * -h 172.19.124.72 + * -p 8081 + * @param args + * @return + */ public static LoadOptions parseOptions(String[] args) { LoadOptions options = new LoadOptions(); JCommander commander = JCommander.newBuilder() - .addObject(options) - .build(); + .addObject(options) + .build(); commander.parse(args); // Print usage and exit if (options.help) { @@ -229,10 +277,10 @@ public static LoadOptions parseOptions(String[] args) { // Check options // Check option "-f" E.checkArgument(!StringUtils.isEmpty(options.file), - "The mapping file must be specified"); + "The mapping file must be specified"); E.checkArgument(options.file.endsWith(Constants.JSON_SUFFIX), - "The mapping file name must be end with %s", - Constants.JSON_SUFFIX); + "The mapping file name must be end with %s", + Constants.JSON_SUFFIX); File mappingFile = new File(options.file); if (!mappingFile.canRead()) { LOG.error("The mapping file must be readable: '{}'", mappingFile); @@ -241,7 +289,7 @@ public static LoadOptions parseOptions(String[] args) { // Check option "-g" E.checkArgument(!StringUtils.isEmpty(options.graph), - "The graph must be specified"); + "The graph must be specified"); // Check option "-h" if (!options.host.startsWith(Constants.HTTP_PREFIX)) { if (options.protocol.equals(HTTP_SCHEMA)) { @@ -252,7 +300,7 @@ public static LoadOptions parseOptions(String[] args) { } // Check option --incremental-mode and --failure-mode E.checkArgument(!(options.incrementalMode && options.failureMode), - "The option --incremental-mode and --failure-mode " + + "The option --incremental-mode and --failure-mode " + "can't be true at same time"); if (options.failureMode) { LOG.info("The failure-mode will scan the entire error file"); @@ -263,17 +311,28 @@ public static LoadOptions parseOptions(String[] args) { return options; } + public void copyBackendStoreInfo(BackendStoreInfo backendStoreInfo){ + E.checkArgument(null != backendStoreInfo,"the backendStoreInfo is null "); + this.edgeTablename=backendStoreInfo.getEdgeTablename(); + this.vertexTablename=backendStoreInfo.getVertexTablename(); + this.hbaseZKParent=backendStoreInfo.getHbaseZKParent(); + this.hbaseZKPort=backendStoreInfo.getHbaseZKPort(); + this.hbaseZKQuorum=backendStoreInfo.getHbaseZKQuorum(); + this.targetHdfsHfilePath=backendStoreInfo.getTargetHdfsHfilePath(); + + } + public static class UrlValidator implements IParameterValidator { @Override public void validate(String name, String value) { String regex = "^((http)(s?)://)?" + - "(([0-9]{1,3}\\.){3}[0-9]{1,3}" + // IP URL - "|" + // Or domain name - "([0-9a-z_!~*'()-]+\\.)*[0-9a-z_!~*'()-]+)$"; + "(([0-9]{1,3}\\.){3}[0-9]{1,3}" + // IP URL + "|" + // Or domain name + "([0-9a-z_!~*'()-]+\\.)*[0-9a-z_!~*'()-]+)$"; if (!value.matches(regex)) { throw new ParameterException(String.format( - "Invalid url value of args '%s': '%s'", name, value)); + "Invalid url value of args '%s': '%s'", name, value)); } } } @@ -288,8 +347,8 @@ public static class ProtocolValidator implements IParameterValidator { public void validate(String name, String value) { if (!SSL_PROTOCOL.contains(value.toLowerCase())) { throw new ParameterException(String.format( - "Invalid --protocol '%s', valid value is %s", - value, SSL_PROTOCOL)); + "Invalid --protocol '%s', valid value is %s", + value, SSL_PROTOCOL)); } } } @@ -301,8 +360,8 @@ public void validate(String name, String value) { File file = new File(value); if (!file.exists() || !file.isDirectory()) { throw new ParameterException(String.format( - "Ensure the directory exists and is indeed a " + - "directory instead of a file: '%s'", value)); + "Ensure the directory exists and is indeed a " + + "directory instead of a file: '%s'", value)); } } } @@ -314,8 +373,8 @@ public void validate(String name, String value) { File file = new File(value); if (!file.exists() || !file.isFile()) { throw new ParameterException(String.format( - "Ensure the file exists and is indeed a file " + - "instead of a directory: '%s'", value)); + "Ensure the file exists and is indeed a file " + + "instead of a directory: '%s'", value)); } } } @@ -327,8 +386,8 @@ public void validate(String name, String value) { int retry = Integer.parseInt(value); if (retry <= 0) { throw new ParameterException(String.format( - "Parameter '%s' should be positive, but got '%s'", - name, value)); + "Parameter '%s' should be positive, but got '%s'", + name, value)); } } } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java new file mode 100644 index 000000000..c049e97ee --- /dev/null +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java @@ -0,0 +1,76 @@ +package com.baidu.hugegraph.loader.mapping; + + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonPropertyOrder({"edge_tablename", "vertex_tablename", "hbase_zookeeper_quorum", "hbase_zookeeper_property_clientPort", "zookeeper_znode_parent"}) +public class BackendStoreInfo { + @JsonProperty("edge_tablename") + private String edgeTablename; + @JsonProperty("vertex_tablename") + private String vertexTablename; + @JsonProperty("hbase_zookeeper_quorum") + private String hbaseZKQuorum; + @JsonProperty("hbase_zookeeper_property_clientPort") + private String hbaseZKPort; + @JsonProperty("zookeeper_znode_parent") + private String hbaseZKParent; + @JsonProperty("target_hdfs_hfile_path") + private String targetHdfsHfilePath; + + public String getTargetHdfsHfilePath() { + return targetHdfsHfilePath; + } + + public void setTargetHdfsHfilePath(String targetHdfsHfilePath) { + this.targetHdfsHfilePath = targetHdfsHfilePath; + } + + + + + public String getEdgeTablename() { + return edgeTablename; + } + + public void setEdgeTablename(String edgeTablename) { + this.edgeTablename = edgeTablename; + } + + public String isVertexTablename() { + return vertexTablename; + } + + public void setVertexTablename(String vertexTablename) { + this.vertexTablename = vertexTablename; + } + + public String getVertexTablename() { + return vertexTablename; + } + + public String getHbaseZKQuorum() { + return hbaseZKQuorum; + } + + public void setHbaseZKQuorum(String hbaseZKQuorum) { + this.hbaseZKQuorum = hbaseZKQuorum; + } + + public String getHbaseZKPort() { + return hbaseZKPort; + } + + public void setHbaseZKPort(String hbaseZKPort) { + this.hbaseZKPort = hbaseZKPort; + } + + public String getHbaseZKParent() { + return hbaseZKParent; + } + + public void setHbaseZKParent(String hbaseZKParent) { + this.hbaseZKParent = hbaseZKParent; + } +} diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java index ff342dd52..60f63dc94 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java @@ -53,22 +53,30 @@ public class LoadMapping implements Checkable { private String version; @JsonProperty("structs") private List structs; + @JsonProperty("backendStoreInfo") + private BackendStoreInfo backendStoreInfo; + + + public BackendStoreInfo getBackendStoreInfo() { + return backendStoreInfo; + } public static LoadMapping of(String filePath) { File file = FileUtils.getFile(filePath); LoadMapping mapping; try { + // 解析struct.json文件，处理输入数据相关信息 String json = FileUtils.readFileToString(file, Constants.CHARSET); mapping = MappingUtil.parse(json); } catch (IOException e) { throw new LoadException("Failed to read mapping mapping file '%s'", - e, filePath); + e, filePath); } catch (IllegalArgumentException e) { throw new LoadException("Failed to parse mapping mapping file '%s'", - e, filePath); + e, filePath); } try { - mapping.check(); +// mapping.check(); } catch (IllegalArgumentException e) { throw new LoadException("Invalid mapping file '%s'", e, filePath); } @@ -81,20 +89,27 @@ public LoadMapping(@JsonProperty("structs") List structs) { this.structs = structs; } + @JsonCreator + public LoadMapping(@JsonProperty("structs") List structs,@JsonProperty("backendStoreInfo") BackendStoreInfo backendStoreInfo) { + this.version = Constants.V2_STRUCT_VERSION; + this.structs = structs; + this.backendStoreInfo=backendStoreInfo; + } + @Override public void check() throws IllegalArgumentException { E.checkArgument(!StringUtils.isEmpty(this.version), - "The version can't be null or empty"); + "The version can't be null or empty"); E.checkArgument(this.version.equals(Constants.V2_STRUCT_VERSION), - "The version must be '%s', but got '%s'", - Constants.V2_STRUCT_VERSION, this.version); + "The version must be '%s', but got '%s'", + Constants.V2_STRUCT_VERSION, this.version); E.checkArgument(!CollectionUtils.isEmpty(this.structs), - "The structs can't be null or empty"); + "The structs can't be null or empty"); this.structs.forEach(InputStruct::check); Set uniqueIds = this.structs.stream().map(InputStruct::id) - .collect(Collectors.toSet()); + .collect(Collectors.toSet()); E.checkArgument(this.structs.size() == uniqueIds.size(), - "The structs cannot contain the same id mapping"); + "The structs cannot contain the same id mapping"); } public List structs() { @@ -122,10 +137,10 @@ public List structsForFailure(LoadOptions options) { String json; try { json = FileUtils.readFileToString(failureFile.headerFile, - charset); + charset); } catch (IOException e) { throw new LoadException("Failed to read header file %s", - failureFile.headerFile); + failureFile.headerFile); } List header = JsonUtil.convertList(json, String.class); source.header(header.toArray(new String[] {})); @@ -143,7 +158,7 @@ public List structsForFailure(LoadOptions options) { private Map groupFailureFiles(File pathDir) { File[] subFiles = pathDir.listFiles(); E.checkArgument(subFiles != null && subFiles.length >= 1, - "Every input struct should have a failure data file, " + + "Every input struct should have a failure data file, " + "and a header file if need it"); Map failureFiles = new LinkedHashMap<>(); for (File subFile : subFiles) { @@ -157,9 +172,9 @@ private Map groupFailureFiles(File pathDir) { failureFile.dataFile = subFile; } else { E.checkArgument(Constants.HEADER_SUFFIX.equals(suffix), - "The failure data file must end with %s or %s", - Constants.FAILURE_SUFFIX, - Constants.HEADER_SUFFIX); + "The failure data file must end with %s or %s", + Constants.FAILURE_SUFFIX, + Constants.HEADER_SUFFIX); failureFile.headerFile = subFile; } failureFiles.put(inputId, failureFile); @@ -174,7 +189,7 @@ public InputStruct struct(String id) { } } throw new IllegalArgumentException(String.format( - "There is no input struct with id '%s'", id)); + "There is no input struct with id '%s'", id)); } private static class FailureFile { diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java new file mode 100644 index 000000000..83945faec --- /dev/null +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java @@ -0,0 +1,184 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.loader.metrics; + +import com.baidu.hugegraph.loader.constant.Constants; +import com.baidu.hugegraph.loader.mapping.EdgeMapping; +import com.baidu.hugegraph.loader.mapping.ElementMapping; +import com.baidu.hugegraph.loader.mapping.InputStruct; +import com.baidu.hugegraph.loader.mapping.VertexMapping; +import org.apache.spark.SparkContext; +import org.apache.spark.util.LongAccumulator; + +import java.io.Serializable; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +public final class LoadDistributeMetrics implements Serializable { + + private final InputStruct struct; + // 分布式写入点边成功导入数量统计 + + private Map vertexDisMetrics; + private Map edgeDisMetrics; + +// private Map vertexParseDisMetrics; +// private Map edgeParseDisMetrics; +// private Map vertexSerDisMetrics; +// private Map edgeSerDisMetrics; +// private Map vertexInsertDisMetrics; +// private Map edgeInsertDisMetrics; + + + + + public LoadDistributeMetrics(InputStruct struct) { + this.struct = struct; + this.vertexDisMetrics = new HashMap<>(); + this.edgeDisMetrics = new HashMap<>(); + for (VertexMapping mapping : struct.vertices()) { + this.vertexDisMetrics.put(mapping.label(), new Metrics()); + } + for (EdgeMapping mapping : struct.edges()) { + this.edgeDisMetrics.put(mapping.label(), new Metrics()); + } + } + + + + + + //初始化分布式metric + public void init(SparkContext sc) { + for (VertexMapping mapping : this.struct.vertices()) { + Metrics metrics = this.vertexDisMetrics.get(mapping.label()); + metrics.insertSuccess=sc.longAccumulator(mapping.label()+ Constants.UNDERLINE_STR+ Constants.LOAD_DATA_INSERT_SUFFIX); + metrics.parseSuccess=sc.longAccumulator(mapping.label()+ Constants.UNDERLINE_STR+ Constants.LOAD_DATA_PARSE_SUFFIX); + + } + for (EdgeMapping mapping : this.struct.edges()) { + Metrics metrics = this.edgeDisMetrics.get(mapping.label()); + metrics.insertSuccess=sc.longAccumulator(mapping.label()+ Constants.UNDERLINE_STR+ Constants.LOAD_DATA_INSERT_SUFFIX); + metrics.parseSuccess=sc.longAccumulator(mapping.label()+ Constants.UNDERLINE_STR+ Constants.LOAD_DATA_PARSE_SUFFIX); + + } + } + + + + + + + + + public void increaseDisVertexParseSuccess(ElementMapping mapping) { + this.disMetrics(mapping).parseSuccess.add(1); + } + public void pluseDisVertexParseSuccess(ElementMapping mapping, Long count) { + this.disMetrics(mapping).parseSuccess.add(count); + } + public void increaseDisVertexInsertSuccess(ElementMapping mapping) { + this.disMetrics(mapping).insertSuccess.add(1); + } + + public void plusDisVertexInsertSuccess(ElementMapping mapping, Long count) { + this.disMetrics(mapping).insertSuccess.add(count); + } + + + + public void increaseDisEdgeParseSuccess(ElementMapping mapping) { + this.disMetrics(mapping).parseSuccess.add(1); + } + public void pluseDisEdgeParseSuccess(ElementMapping mapping, Long count) { + this.disMetrics(mapping).parseSuccess.add(count); + } + public void increaseDisEdgeInsertSuccess(ElementMapping mapping) { + this.disMetrics(mapping).insertSuccess.add(1); + } + + public void plusDisEdgeInsertSuccess(ElementMapping mapping, Long count) { + this.disMetrics(mapping).insertSuccess.add(count); + } + + public Long readVertexInsertSuccess(){ + Long totalCnt=0L; + Collection values = vertexDisMetrics.values(); + for (Metrics metrics : values) { + totalCnt += metrics.insertSuccess(); + } + return totalCnt; + } + + + + public Long readEdgeInsertSuccess(){ + Long totalCnt=0L; + Collection values = edgeDisMetrics.values(); + for (Metrics metrics : values) { + totalCnt += metrics.insertSuccess(); + } + return totalCnt; + } + + private Metrics disMetrics(ElementMapping mapping) { + if (mapping.type().isVertex()) { + return this.vertexDisMetrics.get(mapping.label()); + } else { + return this.edgeDisMetrics.get(mapping.label()); + } + } + public static class Metrics implements Serializable { + +// private final LongAccumulator parseSuccess; +// private final LongAccumulator parseFailure; +// private final LongAccumulator insertSuccess; +// private final LongAccumulator insertFailure; + + private LongAccumulator parseSuccess; + private LongAccumulator parseFailure; + private LongAccumulator insertSuccess; + private LongAccumulator insertFailure; + + public Metrics() { +// this.parseSuccess = new LongAdder(); +// this.parseFailure = new LongAdder(); +// this.insertSuccess = new LongAdder(); +// this.insertFailure = new LongAdder(); + } + + public long parseSuccess() { + return this.parseSuccess.value(); + } + + public long parseFailure() { + return this.parseFailure.value(); + } + + public long insertSuccess() { + return this.insertSuccess.value(); + } + + public long insertFailure() { + return this.insertFailure.value(); + } + } +} diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java index 143dd2f0b..c52a450bf 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java @@ -20,48 +20,47 @@ package com.baidu.hugegraph.loader.spark; import com.baidu.hugegraph.driver.GraphManager; +import com.baidu.hugegraph.driver.HugeClient; +import com.baidu.hugegraph.exception.ServerException; import com.baidu.hugegraph.loader.builder.EdgeBuilder; import com.baidu.hugegraph.loader.builder.ElementBuilder; import com.baidu.hugegraph.loader.builder.VertexBuilder; + +import com.baidu.hugegraph.loader.direct.loader.HBaseDirectLoader; import com.baidu.hugegraph.loader.executor.LoadContext; import com.baidu.hugegraph.loader.executor.LoadOptions; import com.baidu.hugegraph.loader.mapping.EdgeMapping; +import com.baidu.hugegraph.loader.mapping.VertexMapping; import com.baidu.hugegraph.loader.mapping.ElementMapping; import com.baidu.hugegraph.loader.mapping.InputStruct; import com.baidu.hugegraph.loader.mapping.LoadMapping; -import com.baidu.hugegraph.loader.mapping.VertexMapping; +import com.baidu.hugegraph.loader.metrics.LoadDistributeMetrics; +import com.baidu.hugegraph.loader.metrics.LoadMetrics; import com.baidu.hugegraph.loader.source.InputSource; -import com.baidu.hugegraph.loader.source.file.Compression; -import com.baidu.hugegraph.loader.source.file.FileFilter; -import com.baidu.hugegraph.loader.source.file.FileFormat; -import com.baidu.hugegraph.loader.source.file.FileSource; -import com.baidu.hugegraph.loader.source.file.SkippedLine; -import com.baidu.hugegraph.loader.source.jdbc.JDBCSource; +import com.baidu.hugegraph.loader.source.SourceType; +import com.baidu.hugegraph.loader.source.file.*; + +import com.baidu.hugegraph.loader.source.hdfs.HDFSSource; +import com.baidu.hugegraph.loader.util.HugeClientHolder; import com.baidu.hugegraph.loader.util.Printer; +import com.baidu.hugegraph.rest.ClientException; import com.baidu.hugegraph.structure.GraphElement; -import com.baidu.hugegraph.structure.graph.BatchEdgeRequest; -import com.baidu.hugegraph.structure.graph.BatchVertexRequest; -import com.baidu.hugegraph.structure.graph.Edge; import com.baidu.hugegraph.structure.graph.UpdateStrategy; import com.baidu.hugegraph.structure.graph.Vertex; +import com.baidu.hugegraph.structure.graph.Edge; +import com.baidu.hugegraph.structure.graph.BatchEdgeRequest; +import com.baidu.hugegraph.structure.graph.BatchVertexRequest; import com.baidu.hugegraph.util.Log; - -import org.apache.spark.sql.DataFrameReader; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; +import org.apache.spark.SparkConf; +import org.apache.spark.SparkContext; import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.DataFrameReader; import org.slf4j.Logger; - import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; -import scala.collection.JavaConverters; public class HugeGraphSparkLoader implements Serializable { @@ -70,7 +69,9 @@ public class HugeGraphSparkLoader implements Serializable { private final LoadOptions loadOptions; private final Map> builders; - public static void main(String[] args) { + + + public static void main(String[] args) throws Exception { HugeGraphSparkLoader loader; try { loader = new HugeGraphSparkLoader(args); @@ -84,27 +85,99 @@ public static void main(String[] args) { public HugeGraphSparkLoader(String[] args) { this.loadOptions = LoadOptions.parseOptions(args); this.builders = new HashMap<>(); + LOG.info("host: " + this.loadOptions.host + " port: " + this.loadOptions.port + " graph: " + this.loadOptions.graph); + LOG.info("sink mode: " + this.loadOptions.sinkType); } - public void load() { + + + + public void load() throws Exception { + LoadMapping mapping = LoadMapping.of(this.loadOptions.file); + this.loadOptions.copyBackendStoreInfo(mapping.getBackendStoreInfo()); List structs = mapping.structs(); + boolean sinkType = this.loadOptions.sinkType; + - SparkSession session = SparkSession.builder().getOrCreate(); + SparkConf conf = new SparkConf() + .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")// kryo序列化 + .set("spark.kryo.registrationRequired", "true") + .setAppName("spark-gen-hfile") + + .setMaster("local[*]"); + + conf.registerKryoClasses( + new Class[]{org.apache.hadoop.hbase.io.ImmutableBytesWritable.class, + org.apache.hadoop.hbase.KeyValue.class, + org.apache.spark.sql.types.StructType.class, + org.apache.spark.sql.types.StructField[].class, + org.apache.spark.sql.types.StructField.class, + org.apache.spark.sql.types.LongType$.class, + org.apache.spark.sql.types.Metadata.class, + org.apache.spark.sql.types.StringType$.class, + Class.forName("org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage"), + Class.forName("scala.reflect.ClassTag$$anon$1"), + Class.forName("scala.collection.immutable.Set$EmptySet$"), + Class.forName("org.apache.spark.sql.types.DoubleType$") + }); + SparkSession session = SparkSession.builder() + .config(conf) +// .enableHiveSupport() + .getOrCreate(); + + SparkContext sc = session.sparkContext(); + + Long totalInsertSuccess=0L; for (InputStruct struct : structs) { + // 初始化metric + LOG.info("\n init"+ struct.input().asFileSource().path()+" distribute metrics---- \n"); + LoadDistributeMetrics loadDistributeMetrics = new LoadDistributeMetrics(struct); + loadDistributeMetrics.init(sc); + LOG.info(" \n load dat info: \t"+struct.input().asFileSource().path() +"\n start load data ; \n" ); Dataset ds = read(session, struct); - ds.foreachPartition((Iterator p) -> { - LoadContext context = initPartition(this.loadOptions, struct); - p.forEachRemaining((Row row) -> { - loadRow(struct, row, p, context); + if(sinkType){ + LOG.info("\n ------ spark api start load data ------ \n"); + ds.foreachPartition((Iterator p) -> { + LoadContext context = initPartition(this.loadOptions, struct); + p.forEachRemaining((Row row) -> { + loadRow(struct, row, p, context); + }); + context.close(); }); - context.close(); - }); + + }else { + LOG.info("\n spark bulkload gen hfile start \n"); + // gen-hfile + HBaseDirectLoader directLoader = new HBaseDirectLoader(loadOptions, struct,loadDistributeMetrics); + directLoader.bulkload(ds); + + } + collectLoadMetrics(loadDistributeMetrics,totalInsertSuccess); + LOG.info(" \n load data info : \t"+struct.input().asFileSource().path() +"\n load data finish!!!; \n" ); } - session.close(); + + LOG.info("\n ---------导入数据整体任务结束----------------------\n" + + "\n insertSuccess cnt:\t"+totalInsertSuccess+" \n"+ + "\n ---------------------------------------------\n" + ); + + + sc.stop(); session.stop(); + session.close(); + } + + private void collectLoadMetrics(LoadDistributeMetrics loadMetrics,Long totalInsertSuccess){ + + Long edgeInsertSuccess = loadMetrics.readEdgeInsertSuccess(); + Long vertexInsertSuccess = loadMetrics.readVertexInsertSuccess(); + totalInsertSuccess+=edgeInsertSuccess; + totalInsertSuccess+=vertexInsertSuccess; + + } private LoadContext initPartition( LoadOptions loadOptions, InputStruct struct) { LoadContext context = new LoadContext(loadOptions); @@ -115,9 +188,8 @@ private LoadContext initPartition( } for (EdgeMapping edgeMapping : struct.edges()) { this.builders.put(new EdgeBuilder(context, struct, edgeMapping), - new ArrayList<>()); + new ArrayList<>()); } - context.updateSchemaCache(); return context; } @@ -134,9 +206,10 @@ private void loadRow(InputStruct struct, Row row, Iterator p, // Insert List graphElements = builderMap.getValue(); - if (graphElements.size() >= elementMapping.batchSize() || - (!p.hasNext() && graphElements.size() > 0)) { - flush(builderMap, context.client().graph(), this.loadOptions.checkVertex); + if (graphElements.size() > elementMapping.batchSize() || + (!p.hasNext() && graphElements.size() > 0)) { + flush(builderMap, context.client().graph(), + this.loadOptions.checkVertex); } } } @@ -144,22 +217,24 @@ private void loadRow(InputStruct struct, Row row, Iterator p, private Dataset read(SparkSession ss, InputStruct struct) { InputSource input = struct.input(); String charset = input.charset(); + FileSource fileSource = input.asFileSource(); + + String[] header = fileSource.header(); + String delimiter = fileSource.delimiter(); + String path = fileSource.path(); + FileFilter filter = fileSource.filter(); + FileFormat format = fileSource.format(); + String dateFormat = fileSource.dateFormat(); + String timeZone = fileSource.timeZone(); + SkippedLine skippedLine = fileSource.skippedLine(); + Compression compression = fileSource.compression(); + int batchSize = fileSource.batchSize(); + DataFrameReader reader = ss.read(); Dataset ds; switch (input.type()) { case FILE: case HDFS: - FileSource fileSource = input.asFileSource(); - String[] header = fileSource.header(); - String delimiter = fileSource.delimiter(); - String path = fileSource.path(); - FileFilter filter = fileSource.filter(); - FileFormat format = fileSource.format(); - String dateFormat = fileSource.dateFormat(); - String timeZone = fileSource.timeZone(); - SkippedLine skippedLine = fileSource.skippedLine(); - Compression compression = fileSource.compression(); - int batchSize = fileSource.batchSize(); switch (format) { case TEXT: ds = reader.text(path); @@ -172,23 +247,14 @@ private Dataset read(SparkSession ss, InputStruct struct) { break; default: throw new IllegalStateException( - "Unexpected format value: " + format); + "Unexpected format value: " + format); } break; case JDBC: - JDBCSource jdbcSource = (JDBCSource) struct.input(); - String url = jdbcSource.url() + "/" + jdbcSource.database(); - String table = jdbcSource.table(); - String username = jdbcSource.username(); - String password = jdbcSource.password(); - Properties properties = new Properties(); - properties.put("user", username); - properties.put("password", password); - ds = reader.jdbc(url, table, properties); - break; + // TODO: implement jdbc default: throw new AssertionError(String.format( - "Unsupported input source '%s'", input.type())); + "Unsupported input source '%s'", input.type())); } return ds; } @@ -206,26 +272,16 @@ private void parse(Row row, case FILE: case HDFS: FileSource fileSource = struct.input().asFileSource(); - String delimiter = fileSource.delimiter(); elements = builder.build(fileSource.header(), - row.mkString(delimiter).split(delimiter)); + row.mkString() + .split(fileSource.delimiter())); break; case JDBC: - Object[] structFields = JavaConverters.asJavaCollection(row.schema().toList()) - .toArray(); - int len = row.schema().length(); - String[] headers = new String[len]; - Object[] values = new Object[len]; - for (int i = 0; i < len; i++) { - headers[i] = ((StructField) structFields[i]).name(); - values[i] = row.get(i); - } - elements = builder.build(headers, values); - break; + //TODO: implement jdbc default: throw new AssertionError(String.format( - "Unsupported input source '%s'", - struct.input().type())); + "Unsupported input source '%s'", + struct.input().type())); } graphElements.addAll(elements); } @@ -249,15 +305,15 @@ private void flush(Map.Entry> builderMap, BatchVertexRequest.Builder req = new BatchVertexRequest.Builder(); req.vertices((List) (Object) graphElements) - .updatingStrategies(updateStrategyMap) - .createIfNotExist(true); + .updatingStrategies(updateStrategyMap) + .createIfNotExist(true); g.updateVertices(req.build()); } else { BatchEdgeRequest.Builder req = new BatchEdgeRequest.Builder(); req.edges((List) (Object) graphElements) - .updatingStrategies(updateStrategyMap) - .checkVertex(isCheckVertex) - .createIfNotExist(true); + .updatingStrategies(updateStrategyMap) + .checkVertex(isCheckVertex) + .createIfNotExist(true); g.updateEdges(req.build()); } } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/struct/GraphStructV1.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/struct/GraphStructV1.java index ec66497aa..4a8081d8c 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/struct/GraphStructV1.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/struct/GraphStructV1.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.stream.Collectors; +import com.baidu.hugegraph.loader.mapping.BackendStoreInfo; import org.apache.commons.collections.ListUtils; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -56,6 +57,9 @@ public class GraphStructV1 implements Checkable { @JsonProperty("edges") private final List edgeStructs; + @JsonProperty("backendStoreInfo") + private BackendStoreInfo backendStoreInfo; + public GraphStructV1() { this.vertexStructs = new ArrayList<>(); this.edgeStructs = new ArrayList<>(); @@ -76,6 +80,10 @@ public static GraphStructV1 of(LoadContext context) { } } + public BackendStoreInfo getBackendStoreInfo() { + return backendStoreInfo; + } + @Override public void check() throws IllegalArgumentException { LOG.info("Checking vertex mapping descriptions"); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java index f6c9fe529..7848aaa97 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; +import com.baidu.hugegraph.loader.executor.LoadOptions; import org.apache.commons.io.FileUtils; import com.baidu.hugegraph.loader.constant.Constants; @@ -45,6 +46,7 @@ import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.InsertionOrderUtil; import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang.StringUtils; @SuppressWarnings("deprecation") public final class MappingUtil { @@ -64,6 +66,8 @@ public static void write(LoadMapping mapping, String path) { } } + + public static LoadMapping parse(String json) { Map map = JsonUtil.convertMap(json, String.class, Object.class); @@ -129,7 +133,7 @@ private static LoadMapping parseV1(String json) { inputStruct.id(String.valueOf(++id)); inputStructs.add(inputStruct); } - return new LoadMapping(inputStructs); + return new LoadMapping(inputStructs,graphStruct.getBackendStoreInfo()); } private static ElementMapping convertV1ToV2(ElementStructV1 origin) { From cc4130c44ebf5d2ae4a82832a2b013df90ad09ff Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Mon, 7 Nov 2022 17:07:47 +0800 Subject: [PATCH 05/25] =?UTF-8?q?=E6=8C=87=E5=AE=9Ahbase=E7=89=88=E6=9C=AC?= =?UTF-8?q?2.2.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index e1f2eebe4..09431d12a 100644 --- a/pom.xml +++ b/pom.xml @@ -80,6 +80,7 @@ hugegraph ${project.name} ${project.version} + 2.2.3 From 2fd7d1e9d856908bad345631f946e9121b7c06c0 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Mon, 7 Nov 2022 19:38:35 +0800 Subject: [PATCH 06/25] feature: spark-loader bypass server for hbase writing hugegraph-loader fix #357 --- .../assembly/static/bin/get-params.sh | 3 +- hugegraph-loader/pom.xml | 190 ++++++++++-------- .../hugegraph/loader/builder/EdgeBuilder.java | 4 +- .../hugegraph/loader/builder/SchemaCache.java | 1 - .../loader/direct/loader/DirectLoader.java | 29 ++- .../direct/loader/HBaseDirectLoader.java | 61 +++--- .../loader/direct/util/SinkToHBase.java | 30 ++- .../loader/executor/LoadOptions.java | 19 +- .../loader/mapping/BackendStoreInfo.java | 34 +++- .../hugegraph/loader/mapping/LoadMapping.java | 3 +- .../loader/metrics/LoadDistributeMetrics.java | 25 +-- .../loader/spark/HugeGraphSparkLoader.java | 149 ++++++++------ .../hugegraph/loader/util/MappingUtil.java | 4 - 13 files changed, 292 insertions(+), 260 deletions(-) diff --git a/hugegraph-loader/assembly/static/bin/get-params.sh b/hugegraph-loader/assembly/static/bin/get-params.sh index 882e80874..e0302399c 100644 --- a/hugegraph-loader/assembly/static/bin/get-params.sh +++ b/hugegraph-loader/assembly/static/bin/get-params.sh @@ -27,8 +27,7 @@ function get_params() { --incremental-mode | --failure-mode | --batch-insert-threads | --single-insert-threads | \ --max-conn | --max-conn-per-route | --batch-size | --max-parse-errors | --max-insert-errors | \ --timeout | --shutdown-timeout | --retry-times | --retry-interval | --check-vertex | \ - --print-progress | --dry-run | --help | "--sink-type" | \ - "--vertex-partitions" | "--edge-partitions" |"--target-hdfs-conf") + --print-progress | --dry-run | --sink-type | --vertex-partitions | --edge-partitions | --help ) HUGEGRAPH_PARAMS="$HUGEGRAPH_PARAMS $1 $2" shift 2 ;; diff --git a/hugegraph-loader/pom.xml b/hugegraph-loader/pom.xml index 6be79740f..d86f00dbe 100644 --- a/hugegraph-loader/pom.xml +++ b/hugegraph-loader/pom.xml @@ -142,6 +142,111 @@ 30.0-jre + + + org.apache.hbase + hbase-common + ${hbase.version} + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-json + + + + + org.apache.hbase + hbase-client + ${hbase.version} + + + hadoop-auth + org.apache.hadoop + + + + + org.apache.hbase + hbase-shaded-client-byo-hadoop + + ${hbase.version} + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-ipc + + + + + org.apache.hbase + hbase-mapreduce + ${hbase.version} + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.apache.hadoop + hadoop-hdfs + + + hadoop-auth + org.apache.hadoop + + + javax.ws.rs-api + javax.ws.rs + + + jersey-client + org.glassfish.jersey.core + + + jersey-common + org.glassfish.jersey.core + + + jersey-server + org.glassfish.jersey.core + + + + + + org.apache.hadoop @@ -315,85 +420,6 @@ com.sun.jersey jersey-json - - com.sun.jersey.contribs - jersey-guice - - - org.apache.parquet - parquet-hadoop-bundle - - - org.apache.parquet - parquet-column - - - org.apache.thrift - libfb303 - - - com.fasterxml.jackson.core - * - - - - - org.apache.hbase - hbase-common - ${hbase.version} - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-servlet - - - com.sun.jersey - jersey-json - - - - - org.apache.hbase - hbase-client - ${hbase.version} - - - org.apache.hbase - hbase-shaded-client-byo-hadoop - - ${hbase.version} - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - org.apache.hadoop - hadoop-common - - - org.apache.hadoop - hadoop-ipc - - - - - org.apache.hbase - hbase-mapreduce - ${hbase.version} - org.slf4j slf4j-api @@ -416,12 +442,6 @@ - - org.apache.hugegraph - hugegraph-client - 1.0.0 - compile - diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java index 0b2c43ca1..498eb6454 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Set; -import com.baidu.hugegraph.loader.constant.Constants; import com.baidu.hugegraph.loader.executor.LoadContext; import com.baidu.hugegraph.loader.mapping.EdgeMapping; import com.baidu.hugegraph.loader.mapping.InputStruct; @@ -70,6 +69,7 @@ public EdgeBuilder(LoadContext context, InputStruct struct, public EdgeMapping mapping() { return this.mapping; } + @Override public List build(String[] names, Object[] values) { if (this.vertexIdsIndex == null || @@ -110,7 +110,7 @@ public List build(String[] names, Object[] values) { } return edges; } - + @Override public List build( Row row) { String[] names = row.schema().fieldNames(); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java index e02460540..7253e0598 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/SchemaCache.java @@ -46,7 +46,6 @@ public SchemaCache(HugeClient client) { this.propertyKeys = new HashMap<>(); this.vertexLabels = new HashMap<>(); this.edgeLabels = new HashMap<>(); - updateAll(); } @JsonCreator diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java index b66bed7da..1a433da28 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java @@ -1,3 +1,22 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package com.baidu.hugegraph.loader.direct.loader; import com.baidu.hugegraph.loader.builder.EdgeBuilder; @@ -16,12 +35,7 @@ import java.util.LinkedList; import java.util.List; -/** - * 1.build - * 2.ser - * 3.generateFile - * 4.load - */ + public abstract class DirectLoader implements Serializable { LoadOptions loadOptions ; InputStruct struct; @@ -36,7 +50,7 @@ public DirectLoader(LoadOptions loadOptions, - public void bulkload(Dataset ds){ + public final void bulkload(Dataset ds){ JavaPairRDD javaPairRDD = buildVertexAndEdge(ds); String path = generateFiles(javaPairRDD); loadFiles(path); @@ -45,6 +59,7 @@ public void bulkload(Dataset ds){ protected List getElementBuilders(){ LoadContext context = new LoadContext(loadOptions); + context.schemaCache().updateAll(); List buildersForGraphElement = new LinkedList<>(); for (VertexMapping vertexMapping : struct.vertices()) { buildersForGraphElement.add( diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java index fe0f57a47..3617868d3 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java @@ -1,23 +1,37 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package com.baidu.hugegraph.loader.direct.loader; import com.baidu.hugegraph.loader.builder.ElementBuilder; import com.baidu.hugegraph.loader.constant.Constants; - import com.baidu.hugegraph.loader.direct.util.SinkToHBase; import com.baidu.hugegraph.loader.executor.LoadOptions; import com.baidu.hugegraph.loader.mapping.ElementMapping; import com.baidu.hugegraph.loader.mapping.InputStruct; import com.baidu.hugegraph.loader.metrics.LoadDistributeMetrics; -import com.baidu.hugegraph.loader.source.file.FileSource; import com.baidu.hugegraph.loader.util.HugeClientHolder; -import com.baidu.hugegraph.loader.util.LoadUtil; import com.baidu.hugegraph.serializer.direct.HBaseSerializer; import com.baidu.hugegraph.structure.GraphElement; import com.baidu.hugegraph.structure.graph.Edge; import com.baidu.hugegraph.structure.graph.Vertex; import com.baidu.hugegraph.util.Log; -import com.baidu.hugegraph.util.StringUtil; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsShell; @@ -37,7 +51,6 @@ import scala.Tuple2; import java.io.IOException; -import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -59,12 +72,10 @@ public HBaseDirectLoader(LoadOptions loadOptions, InputStruct struct, LoadDistri public String getTableName(){ String tableName = null; - if (struct.edges().size() > 0) {//导入的是边 表名换成 出边表 - System.out.println("this is edge struct: " + struct.edges().size()); + if (struct.edges().size() > 0) { tableName = this.loadOptions.edgeTablename; } else if (struct.vertices().size() > 0) { - System.out.println("this is vertices struct: " + struct.vertices().size()); tableName = this.loadOptions.vertexTablename; } @@ -84,15 +95,9 @@ public JavaPairRDD buildVertexAndEdge(Dataset< @Override public Iterator> call(Iterator rowIterator) throws Exception { - List elementsElement;//存储返回的 GraphElement: Vertex/Edge - // 目标存储序列化器 HBaseSerializer serializer = new HBaseSerializer(HugeClientHolder.create(loadOptions),loadOptions.vertexPartitions,loadOptions.edgePartitions); List buildersForGraphElement = getElementBuilders(); - - /** - * 构建 vertex/edge ---> K-V - */ List> result = new LinkedList<>(); while (rowIterator.hasNext()) { Row row = rowIterator.next(); @@ -118,21 +123,12 @@ String generateFiles(JavaPairRDD buildAndSerRd TableDescriptor tableDescriptor= (TableDescriptor) tuple._2; JavaPairRDD repartitionedRdd = - buildAndSerRdd.repartitionAndSortWithinPartitions(partitioner);//精髓 partition内部做到有序 + buildAndSerRdd.repartitionAndSortWithinPartitions(partitioner); Configuration conf = sinkToHBase.getHBaseConfiguration().get(); Job job = Job.getInstance(conf); - LOG.info("hbasedirectloader HFileOutputFormat2 tableDescriptor >>>"+tableDescriptor.toString()); HFileOutputFormat2.configureIncrementalLoadMap(job, tableDescriptor); conf.set("hbase.mapreduce.hfileoutputformat.table.name", tableDescriptor.getTableName().getNameAsString()); String path= getHFilePath(job.getConfiguration()); - - /** - * 准备bulkload相关配置信息 - */ - /** - * 生成 HFile - */ - LOG.info("hbasedirectloader HFileOutputFormat2 tablename >>>"+conf.get("hbase.mapreduce.hfileoutputformat.table.name")); repartitionedRdd.saveAsNewAPIHadoopFile( path, ImmutableBytesWritable.class, @@ -166,8 +162,6 @@ public String getHFilePath(Configuration conf) throws IOException { @Override public void loadFiles(String path) { - long timeStr = System.currentTimeMillis(); - try { sinkToHBase.loadHfiles(path, getTableName());// BulkLoad HFile to HBase } catch (Exception e) { @@ -193,13 +187,12 @@ private void flushPermission(Configuration conf, String path){ } } - // 构建点边然后序列化为目标存储的所需类型 List> buildAndSer(HBaseSerializer serializer, Row row, List buildersForGraphElement) { - List elementsElement;//存储返回的 GraphElement: Vertex/Edge + List elementsElement; List> result = new LinkedList<>(); - for (ElementBuilder builder : buildersForGraphElement) { // 控制类型 点/边 + for (ElementBuilder builder : buildersForGraphElement) { ElementMapping elementMapping = builder.mapping(); if (elementMapping.skip()) { continue; @@ -223,14 +216,14 @@ List> buildAndSer(HBaseSerializer seri boolean isVertex = builder.mapping().type().isVertex(); if (isVertex) { for (Vertex vertex : (List) (Object) elementsElement) { - LOG.info("vertex already build done >>>" + vertex.toString()); + LOG.debug("vertex already build done >>>" + vertex.toString()); Tuple2 tuple2 = vertexSerialize(serializer,vertex); loadDistributeMetrics.increaseDisVertexInsertSuccess(builder.mapping()); result.add(tuple2); } } else { for (Edge edge : (List) (Object) elementsElement) { - LOG.info("edge already build done >>>" + edge.toString()); + LOG.debug("edge already build done >>>" + edge.toString()); Tuple2 tuple2 = edgeSerialize(serializer,edge); loadDistributeMetrics.increaseDisEdgeInsertSuccess(builder.mapping()); result.add(tuple2); @@ -242,7 +235,7 @@ List> buildAndSer(HBaseSerializer seri } private Tuple2 edgeSerialize(HBaseSerializer serializer, Edge edge) { - LOG.info("edge start serialize >>>" + edge.toString()); + LOG.debug("edge start serialize >>>" + edge.toString()); byte[] rowkey = serializer.getKeyBytes(edge); byte[] values = serializer.getValueBytes(edge); ImmutableBytesWritable rowKey = new ImmutableBytesWritable(); @@ -255,7 +248,7 @@ private Tuple2 edgeSerialize(HBaseSerializer s } private Tuple2 vertexSerialize(HBaseSerializer serializer, Vertex vertex) { - LOG.info("vertex start serialize >>>" + vertex.toString()); + LOG.debug("vertex start serialize >>>" + vertex.toString()); byte[] rowkey = serializer.getKeyBytes(vertex); byte[] values = serializer.getValueBytes(vertex); ImmutableBytesWritable rowKey = new ImmutableBytesWritable(); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java index 159a66019..72dbb5a3e 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java @@ -1,3 +1,22 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package com.baidu.hugegraph.loader.direct.util; import com.baidu.hugegraph.loader.executor.LoadOptions; @@ -34,7 +53,6 @@ public Optional getHBaseConfiguration(){ baseConf.set("hbase.zookeeper.quorum", this.loadOptions.hbaseZKQuorum); baseConf.set("hbase.zookeeper.property.clientPort", this.loadOptions.hbaseZKPort); baseConf.set("zookeeper.znode.parent", this.loadOptions.hbaseZKParent); - return Optional.ofNullable(baseConf); } @@ -49,7 +67,6 @@ private Optional getConnection(){ return Optional.ofNullable(conn); } public Tuple2 getPartitionerByTableName (int numPartitions, String tableName) throws IOException { - LOG.debug("getPartitionerByTableName 接收的tablename 》》》"+tableName); Optional optionalConnection = getConnection(); TableDescriptor descriptor = optionalConnection .get() @@ -62,16 +79,9 @@ public Tuple2 getPartitionerByTableName (int nu public void loadHfiles (String path, String tableName) throws Exception { Connection conn = getConnection().get(); - Table table = conn.getTable(TableName.valueOf(tableName)); Configuration conf = conn.getConfiguration(); - System.out.println("conf >>"+conf.get("dfs.client.failover.proxy.provider.hwyriskmgtcluster")); - System.out.println("conf >>"+conf.get("dfs.ha.namenodes.hwyriskmgtcluster")); - System.out.println("conf >>"+conf.get("dfs.namenode.http-address.hwyriskmgtcluster.nn1")); - System.out.println("conf >>"+conf.get("dfs.namenode.http-address.hwyriskmgtcluster.nn2")); BulkLoadHFilesTool bulkLoadHFilesTool = new BulkLoadHFilesTool(conf); - - //TODO: load HFile to HBase bulkLoadHFilesTool.bulkLoad(table.getName(), new Path(path)); table.close(); conn.close(); @@ -79,7 +89,7 @@ public void loadHfiles (String path, String tableName) throws Exception { } - //精髓 repartitionAndSort + public class IntPartitioner extends Partitioner { private final int numPartitions; public Map, Integer> rangeMap = new HashMap<>(); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java index 5934b34ae..47a2181da 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java @@ -236,9 +236,7 @@ public class LoadOptions implements Serializable { @Parameter(names = {"hbaseZKParent"}, arity = 1, description = "hbaseZKParent") public String hbaseZKParent; - @Parameter(names = {"targetHdfsHfilePath"}, arity = 1, - description = "target hdfs hfile path,need to execute distcp ") - public String targetHdfsHfilePath; + public String workModeString() { @@ -251,19 +249,7 @@ public String workModeString() { } } - /** - * --username admin - * --token admin - * -g hugegraph - * -f /home/data/wwwuser/social-graph-struct.json - * -s /home/data/wwwuser/hugegraph-loader/hugegraph-loader-1.0.0 - * /example/social-graph/social-graph-schema.groovy - * --batch-size 500 - * -h 172.19.124.72 - * -p 8081 - * @param args - * @return - */ + public static LoadOptions parseOptions(String[] args) { LoadOptions options = new LoadOptions(); JCommander commander = JCommander.newBuilder() @@ -318,7 +304,6 @@ public void copyBackendStoreInfo(BackendStoreInfo backendStoreInfo){ this.hbaseZKParent=backendStoreInfo.getHbaseZKParent(); this.hbaseZKPort=backendStoreInfo.getHbaseZKPort(); this.hbaseZKQuorum=backendStoreInfo.getHbaseZKQuorum(); - this.targetHdfsHfilePath=backendStoreInfo.getTargetHdfsHfilePath(); } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java index c049e97ee..5a157ab16 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java @@ -1,10 +1,29 @@ -package com.baidu.hugegraph.loader.mapping; +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.baidu.hugegraph.loader.mapping; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -@JsonPropertyOrder({"edge_tablename", "vertex_tablename", "hbase_zookeeper_quorum", "hbase_zookeeper_property_clientPort", "zookeeper_znode_parent"}) +@JsonPropertyOrder({"edge_tablename", "vertex_tablename", "hbase_zookeeper_quorum", + "hbase_zookeeper_property_clientPort", "zookeeper_znode_parent"}) public class BackendStoreInfo { @JsonProperty("edge_tablename") private String edgeTablename; @@ -16,16 +35,9 @@ public class BackendStoreInfo { private String hbaseZKPort; @JsonProperty("zookeeper_znode_parent") private String hbaseZKParent; - @JsonProperty("target_hdfs_hfile_path") - private String targetHdfsHfilePath; + - public String getTargetHdfsHfilePath() { - return targetHdfsHfilePath; - } - - public void setTargetHdfsHfilePath(String targetHdfsHfilePath) { - this.targetHdfsHfilePath = targetHdfsHfilePath; - } + diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java index 60f63dc94..ec4546828 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java @@ -65,7 +65,6 @@ public static LoadMapping of(String filePath) { File file = FileUtils.getFile(filePath); LoadMapping mapping; try { - // 解析struct.json文件，处理输入数据相关信息 String json = FileUtils.readFileToString(file, Constants.CHARSET); mapping = MappingUtil.parse(json); } catch (IOException e) { @@ -76,7 +75,7 @@ public static LoadMapping of(String filePath) { e, filePath); } try { -// mapping.check(); + mapping.check(); } catch (IllegalArgumentException e) { throw new LoadException("Invalid mapping file '%s'", e, filePath); } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java index 83945faec..ae0ae0443 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java @@ -35,17 +35,10 @@ public final class LoadDistributeMetrics implements Serializable { private final InputStruct struct; - // 分布式写入点边成功导入数量统计 private Map vertexDisMetrics; private Map edgeDisMetrics; - -// private Map vertexParseDisMetrics; -// private Map edgeParseDisMetrics; -// private Map vertexSerDisMetrics; -// private Map edgeSerDisMetrics; -// private Map vertexInsertDisMetrics; -// private Map edgeInsertDisMetrics; + @@ -62,11 +55,7 @@ public LoadDistributeMetrics(InputStruct struct) { } } - - - - - //初始化分布式metric + public void init(SparkContext sc) { for (VertexMapping mapping : this.struct.vertices()) { Metrics metrics = this.vertexDisMetrics.get(mapping.label()); @@ -148,21 +137,13 @@ private Metrics disMetrics(ElementMapping mapping) { } public static class Metrics implements Serializable { -// private final LongAccumulator parseSuccess; -// private final LongAccumulator parseFailure; -// private final LongAccumulator insertSuccess; -// private final LongAccumulator insertFailure; - private LongAccumulator parseSuccess; private LongAccumulator parseFailure; private LongAccumulator insertSuccess; private LongAccumulator insertFailure; public Metrics() { -// this.parseSuccess = new LongAdder(); -// this.parseFailure = new LongAdder(); -// this.insertSuccess = new LongAdder(); -// this.insertFailure = new LongAdder(); + } public long parseSuccess() { diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java index c52a450bf..24fef8375 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java @@ -20,46 +20,53 @@ package com.baidu.hugegraph.loader.spark; import com.baidu.hugegraph.driver.GraphManager; -import com.baidu.hugegraph.driver.HugeClient; -import com.baidu.hugegraph.exception.ServerException; import com.baidu.hugegraph.loader.builder.EdgeBuilder; import com.baidu.hugegraph.loader.builder.ElementBuilder; import com.baidu.hugegraph.loader.builder.VertexBuilder; - import com.baidu.hugegraph.loader.direct.loader.HBaseDirectLoader; import com.baidu.hugegraph.loader.executor.LoadContext; import com.baidu.hugegraph.loader.executor.LoadOptions; import com.baidu.hugegraph.loader.mapping.EdgeMapping; -import com.baidu.hugegraph.loader.mapping.VertexMapping; import com.baidu.hugegraph.loader.mapping.ElementMapping; import com.baidu.hugegraph.loader.mapping.InputStruct; import com.baidu.hugegraph.loader.mapping.LoadMapping; +import com.baidu.hugegraph.loader.mapping.VertexMapping; import com.baidu.hugegraph.loader.metrics.LoadDistributeMetrics; -import com.baidu.hugegraph.loader.metrics.LoadMetrics; import com.baidu.hugegraph.loader.source.InputSource; -import com.baidu.hugegraph.loader.source.SourceType; -import com.baidu.hugegraph.loader.source.file.*; - -import com.baidu.hugegraph.loader.source.hdfs.HDFSSource; -import com.baidu.hugegraph.loader.util.HugeClientHolder; +import com.baidu.hugegraph.loader.source.file.Compression; +import com.baidu.hugegraph.loader.source.file.FileFilter; +import com.baidu.hugegraph.loader.source.file.FileFormat; +import com.baidu.hugegraph.loader.source.file.FileSource; +import com.baidu.hugegraph.loader.source.file.SkippedLine; +import com.baidu.hugegraph.loader.source.jdbc.JDBCSource; import com.baidu.hugegraph.loader.util.Printer; -import com.baidu.hugegraph.rest.ClientException; import com.baidu.hugegraph.structure.GraphElement; -import com.baidu.hugegraph.structure.graph.UpdateStrategy; -import com.baidu.hugegraph.structure.graph.Vertex; -import com.baidu.hugegraph.structure.graph.Edge; import com.baidu.hugegraph.structure.graph.BatchEdgeRequest; import com.baidu.hugegraph.structure.graph.BatchVertexRequest; +import com.baidu.hugegraph.structure.graph.Edge; +import com.baidu.hugegraph.structure.graph.UpdateStrategy; +import com.baidu.hugegraph.structure.graph.Vertex; import com.baidu.hugegraph.util.Log; + import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.Dataset; import org.apache.spark.sql.DataFrameReader; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.util.LongAccumulator; import org.slf4j.Logger; + import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import scala.collection.JavaConverters; public class HugeGraphSparkLoader implements Serializable { @@ -69,9 +76,7 @@ public class HugeGraphSparkLoader implements Serializable { private final LoadOptions loadOptions; private final Map> builders; - - - public static void main(String[] args) throws Exception { + public static void main(String[] args) { HugeGraphSparkLoader loader; try { loader = new HugeGraphSparkLoader(args); @@ -85,14 +90,12 @@ public static void main(String[] args) throws Exception { public HugeGraphSparkLoader(String[] args) { this.loadOptions = LoadOptions.parseOptions(args); this.builders = new HashMap<>(); - LOG.info("host: " + this.loadOptions.host + " port: " + this.loadOptions.port + " graph: " + this.loadOptions.graph); - LOG.info("sink mode: " + this.loadOptions.sinkType); } - public void load() throws Exception { + public void load() { LoadMapping mapping = LoadMapping.of(this.loadOptions.file); this.loadOptions.copyBackendStoreInfo(mapping.getBackendStoreInfo()); @@ -102,35 +105,34 @@ public void load() throws Exception { SparkConf conf = new SparkConf() .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")// kryo序列化 - .set("spark.kryo.registrationRequired", "true") - .setAppName("spark-gen-hfile") - - .setMaster("local[*]"); - - conf.registerKryoClasses( - new Class[]{org.apache.hadoop.hbase.io.ImmutableBytesWritable.class, - org.apache.hadoop.hbase.KeyValue.class, - org.apache.spark.sql.types.StructType.class, - org.apache.spark.sql.types.StructField[].class, - org.apache.spark.sql.types.StructField.class, - org.apache.spark.sql.types.LongType$.class, - org.apache.spark.sql.types.Metadata.class, - org.apache.spark.sql.types.StringType$.class, - Class.forName("org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage"), - Class.forName("scala.reflect.ClassTag$$anon$1"), - Class.forName("scala.collection.immutable.Set$EmptySet$"), - Class.forName("org.apache.spark.sql.types.DoubleType$") - }); + .set("spark.kryo.registrationRequired", "true"); + + try { + conf.registerKryoClasses( + new Class[]{org.apache.hadoop.hbase.io.ImmutableBytesWritable.class, + org.apache.hadoop.hbase.KeyValue.class, + org.apache.spark.sql.types.StructType.class, + StructField[].class, + StructField.class, + org.apache.spark.sql.types.LongType$.class, + org.apache.spark.sql.types.Metadata.class, + org.apache.spark.sql.types.StringType$.class, + Class.forName("org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage"), + Class.forName("scala.reflect.ClassTag$$anon$1"), + Class.forName("scala.collection.immutable.Set$EmptySet$"), + Class.forName("org.apache.spark.sql.types.DoubleType$") + }); + } catch (ClassNotFoundException e) { + LOG.error("spark kryo serialized registration failed"); + } SparkSession session = SparkSession.builder() .config(conf) -// .enableHiveSupport() .getOrCreate(); SparkContext sc = session.sparkContext(); - Long totalInsertSuccess=0L; + LongAccumulator totalInsertSuccess = sc.longAccumulator("totalInsertSuccess"); for (InputStruct struct : structs) { - // 初始化metric LOG.info("\n init"+ struct.input().asFileSource().path()+" distribute metrics---- \n"); LoadDistributeMetrics loadDistributeMetrics = new LoadDistributeMetrics(struct); loadDistributeMetrics.init(sc); @@ -156,28 +158,30 @@ public void load() throws Exception { collectLoadMetrics(loadDistributeMetrics,totalInsertSuccess); LOG.info(" \n load data info : \t"+struct.input().asFileSource().path() +"\n load data finish!!!; \n" ); } - - LOG.info("\n ---------导入数据整体任务结束----------------------\n" + + Long totalInsertSuccessCnt = totalInsertSuccess.value(); + LOG.info("\n ------------The data import task is complete-------------------\n" + "\n insertSuccess cnt:\t"+totalInsertSuccess+" \n"+ "\n ---------------------------------------------\n" ); sc.stop(); - session.stop(); session.close(); + session.stop(); + } - private void collectLoadMetrics(LoadDistributeMetrics loadMetrics,Long totalInsertSuccess){ + private void collectLoadMetrics(LoadDistributeMetrics loadMetrics,LongAccumulator totalInsertSuccess){ Long edgeInsertSuccess = loadMetrics.readEdgeInsertSuccess(); Long vertexInsertSuccess = loadMetrics.readVertexInsertSuccess(); - totalInsertSuccess+=edgeInsertSuccess; - totalInsertSuccess+=vertexInsertSuccess; + totalInsertSuccess.add(edgeInsertSuccess); + totalInsertSuccess.add(vertexInsertSuccess); } + private LoadContext initPartition( LoadOptions loadOptions, InputStruct struct) { LoadContext context = new LoadContext(loadOptions); @@ -190,6 +194,7 @@ private LoadContext initPartition( this.builders.put(new EdgeBuilder(context, struct, edgeMapping), new ArrayList<>()); } + context.updateSchemaCache(); return context; } @@ -206,7 +211,7 @@ private void loadRow(InputStruct struct, Row row, Iterator p, // Insert List graphElements = builderMap.getValue(); - if (graphElements.size() > elementMapping.batchSize() || + if (graphElements.size() >= elementMapping.batchSize() || (!p.hasNext() && graphElements.size() > 0)) { flush(builderMap, context.client().graph(), this.loadOptions.checkVertex); @@ -217,6 +222,11 @@ private void loadRow(InputStruct struct, Row row, Iterator p, private Dataset read(SparkSession ss, InputStruct struct) { InputSource input = struct.input(); String charset = input.charset(); + DataFrameReader reader = ss.read(); + Dataset ds; + switch (input.type()) { + case FILE: + case HDFS: FileSource fileSource = input.asFileSource(); String[] header = fileSource.header(); @@ -229,12 +239,6 @@ private Dataset read(SparkSession ss, InputStruct struct) { SkippedLine skippedLine = fileSource.skippedLine(); Compression compression = fileSource.compression(); int batchSize = fileSource.batchSize(); - - DataFrameReader reader = ss.read(); - Dataset ds; - switch (input.type()) { - case FILE: - case HDFS: switch (format) { case TEXT: ds = reader.text(path); @@ -251,7 +255,16 @@ private Dataset read(SparkSession ss, InputStruct struct) { } break; case JDBC: - // TODO: implement jdbc + JDBCSource jdbcSource = (JDBCSource) struct.input(); + String url = jdbcSource.url() + "/" + jdbcSource.database(); + String table = jdbcSource.table(); + String username = jdbcSource.username(); + String password = jdbcSource.password(); + Properties properties = new Properties(); + properties.put("user", username); + properties.put("password", password); + ds = reader.jdbc(url, table, properties); + break; default: throw new AssertionError(String.format( "Unsupported input source '%s'", input.type())); @@ -272,12 +285,22 @@ private void parse(Row row, case FILE: case HDFS: FileSource fileSource = struct.input().asFileSource(); + String delimiter = fileSource.delimiter(); elements = builder.build(fileSource.header(), - row.mkString() - .split(fileSource.delimiter())); + row.mkString(delimiter).split(delimiter)); break; case JDBC: - //TODO: implement jdbc + Object[] structFields = JavaConverters.asJavaCollection(row.schema().toList()) + .toArray(); + int len = row.schema().length(); + String[] headers = new String[len]; + Object[] values = new Object[len]; + for (int i = 0; i < len; i++) { + headers[i] = ((StructField) structFields[i]).name(); + values[i] = row.get(i); + } + elements = builder.build(headers, values); + break; default: throw new AssertionError(String.format( "Unsupported input source '%s'", diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java index 7848aaa97..a749d842e 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Set; -import com.baidu.hugegraph.loader.executor.LoadOptions; import org.apache.commons.io.FileUtils; import com.baidu.hugegraph.loader.constant.Constants; @@ -46,7 +45,6 @@ import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.InsertionOrderUtil; import com.google.common.collect.ImmutableSet; -import org.apache.commons.lang.StringUtils; @SuppressWarnings("deprecation") public final class MappingUtil { @@ -66,8 +64,6 @@ public static void write(LoadMapping mapping, String path) { } } - - public static LoadMapping parse(String json) { Map map = JsonUtil.convertMap(json, String.class, Object.class); From 5e1e9bb7ddf21f058276a73a3896f722a0dcbedb Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 07:07:21 +0800 Subject: [PATCH 07/25] feature: spark-loader bypass server for hbase writing hugegraph-loader adjust code style fix #357 --- .../scripts/dependency/known-dependencies.txt | 18 +++ .../hugegraph/loader/builder/EdgeBuilder.java | 4 +- .../loader/builder/ElementBuilder.java | 3 +- .../loader/builder/VertexBuilder.java | 1 + .../hugegraph/loader/constant/Constants.java | 2 +- .../loader/direct/loader/DirectLoader.java | 19 +-- .../direct/loader/HBaseDirectLoader.java | 119 +++++++++--------- .../loader/direct/util/SinkToHBase.java | 55 ++++---- .../loader/executor/LoadOptions.java | 15 +-- .../loader/mapping/BackendStoreInfo.java | 10 +- .../loader/mapping/ElementMapping.java | 2 +- .../hugegraph/loader/mapping/LoadMapping.java | 6 +- .../loader/metrics/LoadDistributeMetrics.java | 45 +++---- .../loader/spark/HugeGraphSparkLoader.java | 84 ++++++------- 14 files changed, 194 insertions(+), 189 deletions(-) diff --git a/hugegraph-dist/scripts/dependency/known-dependencies.txt b/hugegraph-dist/scripts/dependency/known-dependencies.txt index e6a798930..6f5f087b4 100644 --- a/hugegraph-dist/scripts/dependency/known-dependencies.txt +++ b/hugegraph-dist/scripts/dependency/known-dependencies.txt @@ -579,3 +579,21 @@ zookeeper-3.4.9.jar zookeeper-3.6.2.jar zookeeper-jute-3.6.2.jar zstd-jni-1.5.0-4.jar +hbase-client-2.2.3.jar +hbase-common-2.2.3.jar +hbase-hadoop-compat-2.2.3.jar +hbase-hadoop2-compat-2.2.3.jar +hbase-http-2.2.3.jar +hbase-mapreduce-2.2.3.jar +hbase-metrics-2.2.3.jar +hbase-metrics-api-2.2.3.jar +hbase-procedure-2.2.3.jar +hbase-protocol-2.2.3.jar +hbase-protocol-shaded-2.2.3.jar +hbase-replication-2.2.3.jar +hbase-server-2.2.3.jar +hbase-shaded-client-byo-hadoop-2.2.3.jar +hbase-shaded-miscellaneous-2.2.1.jar +hbase-shaded-netty-2.2.1.jar +hbase-shaded-protobuf-2.2.1.jar +hbase-zookeeper-2.2.3.jar \ No newline at end of file diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java index 498eb6454..8c8f79dde 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java @@ -112,14 +112,14 @@ public List build(String[] names, Object[] values) { } @Override - public List build( Row row) { + public List build(Row row) { String[] names = row.schema().fieldNames(); Object[] values = new Object[row.size()]; for (int i = 0; i < row.size(); i++) { values[i] = row.get(i); } if (this.vertexIdsIndex == null || - !Arrays.equals(this.lastNames, names)) { + !Arrays.equals(this.lastNames, names)) { this.vertexIdsIndex = this.extractVertexIdsIndex(names); } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/ElementBuilder.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/ElementBuilder.java index 176871366..57c7b3a97 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/ElementBuilder.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/ElementBuilder.java @@ -75,7 +75,8 @@ public ElementBuilder(LoadContext context, InputStruct struct) { public abstract ElementMapping mapping(); public abstract List build(String[] names, Object[] values); - public abstract List build( Row row); + + public abstract List build(Row row); public abstract SchemaLabel schemaLabel(); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java index 64e3f26cf..7f90c6319 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java @@ -51,6 +51,7 @@ public VertexBuilder(LoadContext context, InputStruct struct, public VertexMapping mapping() { return this.mapping; } + @Override public List build(String[] names, Object[] values) { VertexKVPairs kvPairs = this.newKVPairs(this.vertexLabel, diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java index cf8f0852f..fb42312ae 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java @@ -79,7 +79,7 @@ public final class Constants { public static final String CDC_DATA = "data"; public static final String CDC_OP = "op"; - public static final String HBASE_COL_FAMILY="f"; + public static final String HBASE_COL_FAMILY = "f"; public static final String LOAD_DATA_PARSE_SUFFIX = "parse"; public static final String LOAD_DATA_SER_SUFFIX = "ser"; public static final String LOAD_DATA_INSERT_SUFFIX = "insert"; diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java index 1a433da28..bbbc4359d 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java @@ -35,29 +35,23 @@ import java.util.LinkedList; import java.util.List; - public abstract class DirectLoader implements Serializable { - LoadOptions loadOptions ; + LoadOptions loadOptions; InputStruct struct; - public DirectLoader(LoadOptions loadOptions, InputStruct struct) { this.loadOptions = loadOptions; - this.struct=struct; + this.struct = struct; } - - - - public final void bulkload(Dataset ds){ + public final void bulkload(Dataset ds) { JavaPairRDD javaPairRDD = buildVertexAndEdge(ds); String path = generateFiles(javaPairRDD); loadFiles(path); - }; - + } - protected List getElementBuilders(){ + protected List getElementBuilders() { LoadContext context = new LoadContext(loadOptions); context.schemaCache().updateAll(); List buildersForGraphElement = new LinkedList<>(); @@ -73,11 +67,8 @@ protected List getElementBuilders(){ return buildersForGraphElement; } - abstract JavaPairRDD buildVertexAndEdge(Dataset ds); - - abstract String generateFiles(JavaPairRDD buildAndSerRdd); abstract void loadFiles(String path); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java index 3617868d3..89045ed5f 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java @@ -57,19 +57,21 @@ public class HBaseDirectLoader extends DirectLoader { - private SinkToHBase sinkToHBase ; - private LoadDistributeMetrics loadDistributeMetrics;; + private SinkToHBase sinkToHBase; + private LoadDistributeMetrics loadDistributeMetrics; public static final Logger LOG = Log.logger(HBaseDirectLoader.class); - public HBaseDirectLoader(LoadOptions loadOptions, InputStruct struct, LoadDistributeMetrics loadDistributeMetrics) { + public HBaseDirectLoader(LoadOptions loadOptions, + InputStruct struct, + LoadDistributeMetrics loadDistributeMetrics) { super(loadOptions,struct); - this.loadDistributeMetrics=loadDistributeMetrics; - this.sinkToHBase=new SinkToHBase(loadOptions); + this.loadDistributeMetrics = loadDistributeMetrics; + this.sinkToHBase = new SinkToHBase(loadOptions); } - public String getTableName(){ + public String getTableName() { String tableName = null; if (struct.edges().size() > 0) { @@ -82,53 +84,58 @@ public String getTableName(){ return tableName; } - public Integer getTablePartitions(){ - return struct.edges().size() > 0 ? loadOptions.edgePartitions - : loadOptions.vertexPartitions; + public Integer getTablePartitions () { + return struct.edges().size() > 0 ? + loadOptions.edgePartitions : + loadOptions.vertexPartitions; } public JavaPairRDD buildVertexAndEdge(Dataset ds) { LOG.info("buildAndSer start execute >>>>"); - JavaPairRDD tuple2KeyValueJavaPairRDD = ds.toJavaRDD().mapPartitionsToPair( - new PairFlatMapFunction, ImmutableBytesWritable, KeyValue>() { - - @Override - public Iterator> call(Iterator rowIterator) throws Exception { - - - HBaseSerializer serializer = new HBaseSerializer(HugeClientHolder.create(loadOptions),loadOptions.vertexPartitions,loadOptions.edgePartitions); - List buildersForGraphElement = getElementBuilders(); - List> result = new LinkedList<>(); - while (rowIterator.hasNext()) { - Row row = rowIterator.next(); - List> serList = buildAndSer(serializer, row,buildersForGraphElement); - result.addAll(serList); + JavaPairRDD tuple2KeyValueJavaPairRDD = + ds.toJavaRDD().mapPartitionsToPair( + new PairFlatMapFunction, ImmutableBytesWritable, KeyValue>() { + @Override + public Iterator> call( + Iterator rowIterator) throws Exception { + + HBaseSerializer serializer = new HBaseSerializer( + HugeClientHolder.create(loadOptions), + loadOptions.vertexPartitions,loadOptions.edgePartitions); + List buildersForGraphElement = getElementBuilders(); + List> result = + new LinkedList<>(); + while (rowIterator.hasNext()) { + Row row = rowIterator.next(); + List> serList = + buildAndSer(serializer, row,buildersForGraphElement); + result.addAll(serList); + } + serializer.close(); + return result.iterator(); + } } - serializer.close(); - return result.iterator(); - } - } ); return tuple2KeyValueJavaPairRDD; } - @Override String generateFiles(JavaPairRDD buildAndSerRdd) { LOG.info("bulkload start execute>>>"); try { Tuple2 tuple = sinkToHBase.getPartitionerByTableName(getTablePartitions(), getTableName()); - Partitioner partitioner= (Partitioner) tuple._1; - TableDescriptor tableDescriptor= (TableDescriptor) tuple._2; + Partitioner partitioner = (Partitioner) tuple._1; + TableDescriptor tableDescriptor = (TableDescriptor) tuple._2; JavaPairRDD repartitionedRdd = buildAndSerRdd.repartitionAndSortWithinPartitions(partitioner); Configuration conf = sinkToHBase.getHBaseConfiguration().get(); Job job = Job.getInstance(conf); HFileOutputFormat2.configureIncrementalLoadMap(job, tableDescriptor); - conf.set("hbase.mapreduce.hfileoutputformat.table.name", tableDescriptor.getTableName().getNameAsString()); - String path= getHFilePath(job.getConfiguration()); + conf.set("hbase.mapreduce.hfileoutputformat.table.name", + tableDescriptor.getTableName().getNameAsString()); + String path = getHFilePath(job.getConfiguration()); repartitionedRdd.saveAsNewAPIHadoopFile( path, ImmutableBytesWritable.class, @@ -147,19 +154,18 @@ String generateFiles(JavaPairRDD buildAndSerRd } - public String getHFilePath(Configuration conf) throws IOException { + public String getHFilePath (Configuration conf) throws IOException { FileSystem fs = FileSystem.get(conf); long timeStr = System.currentTimeMillis(); - String pathStr = fs.getWorkingDirectory().toString() + "/hfile-gen" + "/" + timeStr+ "/";//HFile 存储路径 + String pathStr = fs.getWorkingDirectory().toString() + "/hfile-gen" + "/" + timeStr + "/"; Path hfileGenPath = new Path(pathStr); - if(fs.exists(hfileGenPath)){ + if (fs.exists(hfileGenPath)) { LOG.info("\n delete hfile path \n"); fs.delete(hfileGenPath,true); } return pathStr; } - @Override public void loadFiles(String path) { try { @@ -167,32 +173,30 @@ public void loadFiles(String path) { } catch (Exception e) { e.printStackTrace(); } - - } - private void flushPermission(Configuration conf, String path){ + private void flushPermission (Configuration conf, String path) { FsShell shell = new FsShell(conf); try { LOG.info("shell start execute"); shell.run(new String[]{"-chmod", "-R", "777", path}); shell.close(); } catch (Exception e) { - LOG.error("Couldnt change the file permissions " + e - + " Please run command:" - + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles " - + path + " '" - + "test" + "'\n" - + " to load generated HFiles into HBase table"); + LOG.error("Couldnt change the file permissions " + e + + " Please run command:" + + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles " + path + + " '" + "test" + "'\n" + " to load generated HFiles into HBase table"); } } - List> buildAndSer(HBaseSerializer serializer, Row row, List buildersForGraphElement) { + List> buildAndSer (HBaseSerializer serializer, + Row row, + List builders) { List elementsElement; List> result = new LinkedList<>(); - for (ElementBuilder builder : buildersForGraphElement) { + for (ElementBuilder builder : builders) { ElementMapping elementMapping = builder.mapping(); if (elementMapping.skip()) { continue; @@ -216,15 +220,17 @@ List> buildAndSer(HBaseSerializer seri boolean isVertex = builder.mapping().type().isVertex(); if (isVertex) { for (Vertex vertex : (List) (Object) elementsElement) { - LOG.debug("vertex already build done >>>" + vertex.toString()); - Tuple2 tuple2 = vertexSerialize(serializer,vertex); + LOG.debug("vertex already build done " + vertex.toString()); + Tuple2 tuple2 = + vertexSerialize(serializer,vertex); loadDistributeMetrics.increaseDisVertexInsertSuccess(builder.mapping()); result.add(tuple2); } } else { for (Edge edge : (List) (Object) elementsElement) { - LOG.debug("edge already build done >>>" + edge.toString()); - Tuple2 tuple2 = edgeSerialize(serializer,edge); + LOG.debug("edge already build done " + edge.toString()); + Tuple2 tuple2 = + edgeSerialize(serializer,edge); loadDistributeMetrics.increaseDisEdgeInsertSuccess(builder.mapping()); result.add(tuple2); @@ -234,8 +240,9 @@ List> buildAndSer(HBaseSerializer seri return result; } - private Tuple2 edgeSerialize(HBaseSerializer serializer, Edge edge) { - LOG.debug("edge start serialize >>>" + edge.toString()); + private Tuple2 edgeSerialize (HBaseSerializer serializer, + Edge edge) { + LOG.debug("edge start serialize " + edge.toString()); byte[] rowkey = serializer.getKeyBytes(edge); byte[] values = serializer.getValueBytes(edge); ImmutableBytesWritable rowKey = new ImmutableBytesWritable(); @@ -247,8 +254,9 @@ private Tuple2 edgeSerialize(HBaseSerializer s return new Tuple2<>(rowKey,keyValue); } - private Tuple2 vertexSerialize(HBaseSerializer serializer, Vertex vertex) { - LOG.debug("vertex start serialize >>>" + vertex.toString()); + private Tuple2 vertexSerialize (HBaseSerializer serializer, + Vertex vertex) { + LOG.debug("vertex start serialize " + vertex.toString()); byte[] rowkey = serializer.getKeyBytes(vertex); byte[] values = serializer.getValueBytes(vertex); ImmutableBytesWritable rowKey = new ImmutableBytesWritable(); @@ -260,5 +268,4 @@ private Tuple2 vertexSerialize(HBaseSerializer return new Tuple2<>(rowKey,keyValue); } - } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java index 72dbb5a3e..8a437f9c6 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java @@ -25,7 +25,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool; import org.apache.hadoop.hbase.util.Bytes; @@ -33,22 +36,26 @@ import org.apache.spark.Partitioner; import org.slf4j.Logger; import scala.Tuple2; - +import org.apache.hadoop.hbase.client.HRegionLocator; +import java.util.ArrayList; +import java.util.Arrays; import java.io.IOException; +import java.util.HashMap; import java.io.Serializable; -import java.util.*; - +import java.util.List; +import java.util.Map; +import java.util.Optional; public class SinkToHBase implements Serializable { + private LoadOptions loadOptions; public static final Logger LOG = Log.logger(SinkToHBase.class); public SinkToHBase(LoadOptions loadOptions) { - this.loadOptions=loadOptions; + this.loadOptions = loadOptions; } - - public Optional getHBaseConfiguration(){ + public Optional getHBaseConfiguration() { Configuration baseConf = HBaseConfiguration.create(); baseConf.set("hbase.zookeeper.quorum", this.loadOptions.hbaseZKQuorum); baseConf.set("hbase.zookeeper.property.clientPort", this.loadOptions.hbaseZKPort); @@ -56,9 +63,9 @@ public Optional getHBaseConfiguration(){ return Optional.ofNullable(baseConf); } - private Optional getConnection(){ + private Optional getConnection() { Optional baseConf = getHBaseConfiguration(); - Connection conn=null; + Connection conn = null; try { conn = ConnectionFactory.createConnection(baseConf.get()); } catch (IOException e) { @@ -66,15 +73,19 @@ private Optional getConnection(){ } return Optional.ofNullable(conn); } - public Tuple2 getPartitionerByTableName (int numPartitions, String tableName) throws IOException { + + public Tuple2 + getPartitionerByTableName (int numPartitions, String tableName) throws IOException { Optional optionalConnection = getConnection(); TableDescriptor descriptor = optionalConnection .get() .getTable(TableName.valueOf(tableName)) .getDescriptor(); - LOG.debug("getPartitionerByTableName get TableDescriptor 》》》"+descriptor.getTableName()); + LOG.debug("getPartitionerByTableName get TableDescriptor " + + descriptor.getTableName()); optionalConnection.get().close(); - return new Tuple2(new IntPartitioner(numPartitions, tableName),descriptor); + return new Tuple2( + new IntPartitioner(numPartitions, tableName),descriptor); } public void loadHfiles (String path, String tableName) throws Exception { @@ -88,8 +99,6 @@ public void loadHfiles (String path, String tableName) throws Exception { } - - public class IntPartitioner extends Partitioner { private final int numPartitions; public Map, Integer> rangeMap = new HashMap<>(); @@ -101,19 +110,15 @@ public IntPartitioner(int numPartitions, String tableName) throws IOException { this.tableName = tableName; } - private Map, Integer> getRangeMap(String tableName) throws IOException { Connection conn = getConnection().get(); - - HRegionLocator locator = (HRegionLocator) conn.getRegionLocator(TableName.valueOf(tableName)); - + HRegionLocator locator = + (HRegionLocator) conn.getRegionLocator(TableName.valueOf(tableName)); Pair startEndKeys = locator.getStartEndKeys(); - Map, Integer> rangeMap = new HashMap<>(); for (int i = 0; i < startEndKeys.getFirst().length; i++) { String startKey = Bytes.toString(startEndKeys.getFirst()[i]); String endKey = Bytes.toString(startEndKeys.getSecond()[i]); - rangeMap.put(new ArrayList<>(Arrays.asList(startKey, endKey)), i); } conn.close(); @@ -138,17 +143,17 @@ public int getPartition(Object key) { String keyString = Bytes.toString(immutableBytesWritableKey.get()); for (List range : rangeMap.keySet()) { - if (keyString.compareToIgnoreCase(range.get(0)) >= 0 - && ((keyString.compareToIgnoreCase(range.get(1)) < 0) - || range.get(1).equals(""))) { + if (keyString.compareToIgnoreCase(range.get(0)) >= 0 && + ((keyString.compareToIgnoreCase(range.get(1)) < 0) || + range.get(1).equals(""))) { return rangeMap.get(range); } } LOG.error("Didn't find proper key in rangeMap, so returning 0 ..."); return 0; } catch (Exception e) { - LOG.error("When trying to get partitionID, " - + "encountered exception: " + e + "\t key = " + key); + LOG.error("When trying to get partitionID, encountered exception: " + e + + "\t key = " + key); return 0; } } else { diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java index 47a2181da..aa7196220 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java @@ -236,8 +236,6 @@ public class LoadOptions implements Serializable { @Parameter(names = {"hbaseZKParent"}, arity = 1, description = "hbaseZKParent") public String hbaseZKParent; - - public String workModeString() { if (this.incrementalMode) { @@ -249,7 +247,6 @@ public String workModeString() { } } - public static LoadOptions parseOptions(String[] args) { LoadOptions options = new LoadOptions(); JCommander commander = JCommander.newBuilder() @@ -297,13 +294,13 @@ public static LoadOptions parseOptions(String[] args) { return options; } - public void copyBackendStoreInfo(BackendStoreInfo backendStoreInfo){ + public void copyBackendStoreInfo (BackendStoreInfo backendStoreInfo) { E.checkArgument(null != backendStoreInfo,"the backendStoreInfo is null "); - this.edgeTablename=backendStoreInfo.getEdgeTablename(); - this.vertexTablename=backendStoreInfo.getVertexTablename(); - this.hbaseZKParent=backendStoreInfo.getHbaseZKParent(); - this.hbaseZKPort=backendStoreInfo.getHbaseZKPort(); - this.hbaseZKQuorum=backendStoreInfo.getHbaseZKQuorum(); + this.edgeTablename = backendStoreInfo.getEdgeTablename(); + this.vertexTablename = backendStoreInfo.getVertexTablename(); + this.hbaseZKParent = backendStoreInfo.getHbaseZKParent(); + this.hbaseZKPort = backendStoreInfo.getHbaseZKPort(); + this.hbaseZKQuorum = backendStoreInfo.getHbaseZKQuorum(); } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java index 5a157ab16..638ca011d 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -@JsonPropertyOrder({"edge_tablename", "vertex_tablename", "hbase_zookeeper_quorum", - "hbase_zookeeper_property_clientPort", "zookeeper_znode_parent"}) +@JsonPropertyOrder({"edge_tablename", "vertex_tablename", "hbase_zookeeper_quorum", + "hbase_zookeeper_property_clientPort", "zookeeper_znode_parent"}) public class BackendStoreInfo { @JsonProperty("edge_tablename") private String edgeTablename; @@ -35,12 +35,6 @@ public class BackendStoreInfo { private String hbaseZKPort; @JsonProperty("zookeeper_znode_parent") private String hbaseZKParent; - - - - - - public String getEdgeTablename() { return edgeTablename; diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/ElementMapping.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/ElementMapping.java index b5ef9acf7..c23413e3d 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/ElementMapping.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/ElementMapping.java @@ -178,7 +178,7 @@ public Object mappingValue(String fieldName, String rawValue) { } public long batchSize() { - return this.batchSize; + return this.batchSize; } public Set selectedFields() { diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java index ec4546828..0e52f5def 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java @@ -56,7 +56,6 @@ public class LoadMapping implements Checkable { @JsonProperty("backendStoreInfo") private BackendStoreInfo backendStoreInfo; - public BackendStoreInfo getBackendStoreInfo() { return backendStoreInfo; } @@ -89,10 +88,11 @@ public LoadMapping(@JsonProperty("structs") List structs) { } @JsonCreator - public LoadMapping(@JsonProperty("structs") List structs,@JsonProperty("backendStoreInfo") BackendStoreInfo backendStoreInfo) { + public LoadMapping(@JsonProperty("structs") List structs, + @JsonProperty("backendStoreInfo") BackendStoreInfo backendStoreInfo) { this.version = Constants.V2_STRUCT_VERSION; this.structs = structs; - this.backendStoreInfo=backendStoreInfo; + this.backendStoreInfo = backendStoreInfo; } @Override diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java index ae0ae0443..1f8690511 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java @@ -35,13 +35,8 @@ public final class LoadDistributeMetrics implements Serializable { private final InputStruct struct; - private Map vertexDisMetrics; private Map edgeDisMetrics; - - - - public LoadDistributeMetrics(InputStruct struct) { this.struct = struct; @@ -55,35 +50,34 @@ public LoadDistributeMetrics(InputStruct struct) { } } - public void init(SparkContext sc) { for (VertexMapping mapping : this.struct.vertices()) { Metrics metrics = this.vertexDisMetrics.get(mapping.label()); - metrics.insertSuccess=sc.longAccumulator(mapping.label()+ Constants.UNDERLINE_STR+ Constants.LOAD_DATA_INSERT_SUFFIX); - metrics.parseSuccess=sc.longAccumulator(mapping.label()+ Constants.UNDERLINE_STR+ Constants.LOAD_DATA_PARSE_SUFFIX); + metrics.insertSuccess = sc.longAccumulator(mapping.label() + + Constants.UNDERLINE_STR + Constants.LOAD_DATA_INSERT_SUFFIX); + metrics.parseSuccess = sc.longAccumulator(mapping.label() + + Constants.UNDERLINE_STR + Constants.LOAD_DATA_PARSE_SUFFIX); } for (EdgeMapping mapping : this.struct.edges()) { Metrics metrics = this.edgeDisMetrics.get(mapping.label()); - metrics.insertSuccess=sc.longAccumulator(mapping.label()+ Constants.UNDERLINE_STR+ Constants.LOAD_DATA_INSERT_SUFFIX); - metrics.parseSuccess=sc.longAccumulator(mapping.label()+ Constants.UNDERLINE_STR+ Constants.LOAD_DATA_PARSE_SUFFIX); + metrics.insertSuccess = sc.longAccumulator(mapping.label() + + Constants.UNDERLINE_STR + Constants.LOAD_DATA_INSERT_SUFFIX); + metrics.parseSuccess = sc.longAccumulator(mapping.label() + + Constants.UNDERLINE_STR + Constants.LOAD_DATA_PARSE_SUFFIX); } } - - - - - - - public void increaseDisVertexParseSuccess(ElementMapping mapping) { + this.disMetrics(mapping).parseSuccess.add(1); } + public void pluseDisVertexParseSuccess(ElementMapping mapping, Long count) { this.disMetrics(mapping).parseSuccess.add(count); } + public void increaseDisVertexInsertSuccess(ElementMapping mapping) { this.disMetrics(mapping).insertSuccess.add(1); } @@ -92,15 +86,17 @@ public void plusDisVertexInsertSuccess(ElementMapping mapping, Long count) { this.disMetrics(mapping).insertSuccess.add(count); } - - public void increaseDisEdgeParseSuccess(ElementMapping mapping) { + this.disMetrics(mapping).parseSuccess.add(1); } + public void pluseDisEdgeParseSuccess(ElementMapping mapping, Long count) { this.disMetrics(mapping).parseSuccess.add(count); } + public void increaseDisEdgeInsertSuccess(ElementMapping mapping) { + this.disMetrics(mapping).insertSuccess.add(1); } @@ -108,8 +104,8 @@ public void plusDisEdgeInsertSuccess(ElementMapping mapping, Long count) { this.disMetrics(mapping).insertSuccess.add(count); } - public Long readVertexInsertSuccess(){ - Long totalCnt=0L; + public Long readVertexInsertSuccess() { + Long totalCnt = 0L; Collection values = vertexDisMetrics.values(); for (Metrics metrics : values) { totalCnt += metrics.insertSuccess(); @@ -117,10 +113,8 @@ public Long readVertexInsertSuccess(){ return totalCnt; } - - - public Long readEdgeInsertSuccess(){ - Long totalCnt=0L; + public Long readEdgeInsertSuccess() { + Long totalCnt = 0L; Collection values = edgeDisMetrics.values(); for (Metrics metrics : values) { totalCnt += metrics.insertSuccess(); @@ -135,6 +129,7 @@ private Metrics disMetrics(ElementMapping mapping) { return this.edgeDisMetrics.get(mapping.label()); } } + public static class Metrics implements Serializable { private LongAccumulator parseSuccess; diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java index 24fef8375..4a76fd65d 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java @@ -65,14 +65,11 @@ import java.util.List; import java.util.Map; import java.util.Properties; - import scala.collection.JavaConverters; - public class HugeGraphSparkLoader implements Serializable { public static final Logger LOG = Log.logger(HugeGraphSparkLoader.class); - private final LoadOptions loadOptions; private final Map> builders; @@ -92,35 +89,32 @@ public HugeGraphSparkLoader(String[] args) { this.builders = new HashMap<>(); } - - - public void load() { LoadMapping mapping = LoadMapping.of(this.loadOptions.file); this.loadOptions.copyBackendStoreInfo(mapping.getBackendStoreInfo()); List structs = mapping.structs(); boolean sinkType = this.loadOptions.sinkType; - - SparkConf conf = new SparkConf() .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")// kryo序列化 .set("spark.kryo.registrationRequired", "true"); - try { - conf.registerKryoClasses( - new Class[]{org.apache.hadoop.hbase.io.ImmutableBytesWritable.class, - org.apache.hadoop.hbase.KeyValue.class, - org.apache.spark.sql.types.StructType.class, - StructField[].class, - StructField.class, - org.apache.spark.sql.types.LongType$.class, - org.apache.spark.sql.types.Metadata.class, - org.apache.spark.sql.types.StringType$.class, - Class.forName("org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage"), - Class.forName("scala.reflect.ClassTag$$anon$1"), - Class.forName("scala.collection.immutable.Set$EmptySet$"), - Class.forName("org.apache.spark.sql.types.DoubleType$") + conf.registerKryoClasses ( + new Class[] + { + org.apache.hadoop.hbase.io.ImmutableBytesWritable.class, + org.apache.hadoop.hbase.KeyValue.class, + org.apache.spark.sql.types.StructType.class, + StructField[].class, + StructField.class, + org.apache.spark.sql.types.LongType$.class, + org.apache.spark.sql.types.Metadata.class, + org.apache.spark.sql.types.StringType$.class, + Class.forName( + "org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage"), + Class.forName("scala.reflect.ClassTag$$anon$1"), + Class.forName("scala.collection.immutable.Set$EmptySet$"), + Class.forName("org.apache.spark.sql.types.DoubleType$") }); } catch (ClassNotFoundException e) { LOG.error("spark kryo serialized registration failed"); @@ -133,12 +127,14 @@ public void load() { LongAccumulator totalInsertSuccess = sc.longAccumulator("totalInsertSuccess"); for (InputStruct struct : structs) { - LOG.info("\n init"+ struct.input().asFileSource().path()+" distribute metrics---- \n"); + LOG.info("\n init" + struct.input().asFileSource().path() + + " distribute metrics---- \n"); LoadDistributeMetrics loadDistributeMetrics = new LoadDistributeMetrics(struct); loadDistributeMetrics.init(sc); - LOG.info(" \n load dat info: \t"+struct.input().asFileSource().path() +"\n start load data ; \n" ); + LOG.info("\n load dat info: \t" + struct.input().asFileSource().path() + + "\n start load data ; \n"); Dataset ds = read(session, struct); - if(sinkType){ + if (sinkType) { LOG.info("\n ------ spark api start load data ------ \n"); ds.foreachPartition((Iterator p) -> { LoadContext context = initPartition(this.loadOptions, struct); @@ -148,32 +144,32 @@ public void load() { context.close(); }); - }else { + } else { LOG.info("\n spark bulkload gen hfile start \n"); // gen-hfile - HBaseDirectLoader directLoader = new HBaseDirectLoader(loadOptions, struct,loadDistributeMetrics); + HBaseDirectLoader directLoader = new HBaseDirectLoader(loadOptions, + struct,loadDistributeMetrics); directLoader.bulkload(ds); } collectLoadMetrics(loadDistributeMetrics,totalInsertSuccess); - LOG.info(" \n load data info : \t"+struct.input().asFileSource().path() +"\n load data finish!!!; \n" ); + LOG.info(" \n load data info : \t" + struct.input().asFileSource().path() + + "\n load data finish!!!; \n"); } Long totalInsertSuccessCnt = totalInsertSuccess.value(); LOG.info("\n ------------The data import task is complete-------------------\n" + - "\n insertSuccess cnt:\t"+totalInsertSuccess+" \n"+ + "\n insertSuccess cnt:\t" + totalInsertSuccess + " \n" + "\n ---------------------------------------------\n" ); - sc.stop(); session.close(); session.stop(); - } - - private void collectLoadMetrics(LoadDistributeMetrics loadMetrics,LongAccumulator totalInsertSuccess){ + private void collectLoadMetrics (LoadDistributeMetrics loadMetrics, + LongAccumulator totalInsertSuccess) { Long edgeInsertSuccess = loadMetrics.readEdgeInsertSuccess(); Long vertexInsertSuccess = loadMetrics.readVertexInsertSuccess(); @@ -227,18 +223,18 @@ private Dataset read(SparkSession ss, InputStruct struct) { switch (input.type()) { case FILE: case HDFS: - FileSource fileSource = input.asFileSource(); + FileSource fileSource = input.asFileSource(); - String[] header = fileSource.header(); - String delimiter = fileSource.delimiter(); - String path = fileSource.path(); - FileFilter filter = fileSource.filter(); - FileFormat format = fileSource.format(); - String dateFormat = fileSource.dateFormat(); - String timeZone = fileSource.timeZone(); - SkippedLine skippedLine = fileSource.skippedLine(); - Compression compression = fileSource.compression(); - int batchSize = fileSource.batchSize(); + String[] header = fileSource.header(); + String delimiter = fileSource.delimiter(); + String path = fileSource.path(); + FileFilter filter = fileSource.filter(); + FileFormat format = fileSource.format(); + String dateFormat = fileSource.dateFormat(); + String timeZone = fileSource.timeZone(); + SkippedLine skippedLine = fileSource.skippedLine(); + Compression compression = fileSource.compression(); + int batchSize = fileSource.batchSize(); switch (format) { case TEXT: ds = reader.text(path); From f0d391a402336906d04e47913a9e38dcf872eef0 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 10:37:55 +0800 Subject: [PATCH 08/25] feature: spark-loader bypass server for hbase writing hugegraph-loader fix third-party dependencies error fix #357 --- hugegraph-loader/pom.xml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/hugegraph-loader/pom.xml b/hugegraph-loader/pom.xml index d86f00dbe..687edc7b5 100644 --- a/hugegraph-loader/pom.xml +++ b/hugegraph-loader/pom.xml @@ -242,7 +242,10 @@ jersey-server org.glassfish.jersey.core - + + com.lmax + disruptor + @@ -380,6 +383,10 @@ org.apache.hadoop hadoop-hdfs + + org.apache.hadoop + hadoop-distcp + org.apache.hadoop hadoop-yarn-registry @@ -400,6 +407,14 @@ org.apache.calcite calcite-druid + + org.jamon + jamon-runtime + + + org.glassfish + javax.el + org.codehaus.groovy groovy-all @@ -440,6 +455,14 @@ org.apache.hadoop hadoop-hdfs + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-core + From 2422c3f881a936ff1b92af35c6d8ef424b6c934b Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 11:45:03 +0800 Subject: [PATCH 09/25] feature: spark-loader bypass server for hbase writing hugegraph-loader fix third-party dependencies error fix #357 --- hugegraph-loader/pom.xml | 42 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/hugegraph-loader/pom.xml b/hugegraph-loader/pom.xml index 687edc7b5..0c82288eb 100644 --- a/hugegraph-loader/pom.xml +++ b/hugegraph-loader/pom.xml @@ -160,6 +160,10 @@ com.sun.jersey jersey-json + + org.apache.htrace + htrace-core4 + @@ -167,6 +171,15 @@ hbase-client ${hbase.version} + + + org.jruby.joni + joni + + + org.jruby.jcodings + jcodings + hadoop-auth org.apache.hadoop @@ -202,6 +215,22 @@ hbase-mapreduce ${hbase.version} + + org.glassfish.web + javax.servlet.jsp + + + javax.servlet.jsp + javax.servlet.jsp-api + + + org.glassfish + javax.el + + + org.jamon + jamon-runtime + org.slf4j slf4j-api @@ -246,7 +275,10 @@ com.lmax disruptor - + + org.apache.hadoop + hadoop-distcp + @@ -359,6 +391,10 @@ hive-exec 3.1.3 + + org.apache.parquet + parquet-hadoop-bundle + log4j log4j @@ -463,6 +499,10 @@ com.fasterxml.jackson.core jackson-core + + org.apache.thrift + libfb303 + From b9fb58c55e0451b139ce0da10f093a4ebfc9b7a2 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 11:46:56 +0800 Subject: [PATCH 10/25] feature: spark-loader bypass server for hbase writing hugegraph-loader fix third-party dependencies error fix #357 --- hugegraph-loader/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hugegraph-loader/pom.xml b/hugegraph-loader/pom.xml index 0c82288eb..44fd2916e 100644 --- a/hugegraph-loader/pom.xml +++ b/hugegraph-loader/pom.xml @@ -148,6 +148,10 @@ hbase-common ${hbase.version} + + com.github.stephenc.findbugs + findbugs-annotations + com.sun.jersey jersey-server From 077b3159d907e05bb919e0b48a0d9d40e9a7a040 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 13:00:59 +0800 Subject: [PATCH 11/25] feature: spark-loader bypass server for hbase writing hugegraph-loader fix third-party dependencies error test 3rd check fix #357 --- hugegraph-loader/pom.xml | 49 ---------------------------------------- 1 file changed, 49 deletions(-) diff --git a/hugegraph-loader/pom.xml b/hugegraph-loader/pom.xml index 44fd2916e..eff34e536 100644 --- a/hugegraph-loader/pom.xml +++ b/hugegraph-loader/pom.xml @@ -164,10 +164,6 @@ com.sun.jersey jersey-json - - org.apache.htrace - htrace-core4 - @@ -175,15 +171,6 @@ hbase-client ${hbase.version} - - - org.jruby.joni - joni - - - org.jruby.jcodings - jcodings - hadoop-auth org.apache.hadoop @@ -219,22 +206,6 @@ hbase-mapreduce ${hbase.version} - - org.glassfish.web - javax.servlet.jsp - - - javax.servlet.jsp - javax.servlet.jsp-api - - - org.glassfish - javax.el - - - org.jamon - jamon-runtime - org.slf4j slf4j-api @@ -275,10 +246,6 @@ jersey-server org.glassfish.jersey.core - - com.lmax - disruptor - org.apache.hadoop hadoop-distcp @@ -395,10 +362,6 @@ hive-exec 3.1.3 - - org.apache.parquet - parquet-hadoop-bundle - log4j log4j @@ -447,14 +410,6 @@ org.apache.calcite calcite-druid - - org.jamon - jamon-runtime - - - org.glassfish - javax.el - org.codehaus.groovy groovy-all @@ -503,10 +458,6 @@ com.fasterxml.jackson.core jackson-core - - org.apache.thrift - libfb303 - From e73944fb72ba330c25f2988fb3bb881090960089 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 13:11:53 +0800 Subject: [PATCH 12/25] feature: spark-loader bypass server for hbase writing hugegraph-loader fix third-party dependencies error add extra jars to known-dependencies.txt fix #357 --- .../scripts/dependency/known-dependencies.txt | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/hugegraph-dist/scripts/dependency/known-dependencies.txt b/hugegraph-dist/scripts/dependency/known-dependencies.txt index 6f5f087b4..6ab318191 100644 --- a/hugegraph-dist/scripts/dependency/known-dependencies.txt +++ b/hugegraph-dist/scripts/dependency/known-dependencies.txt @@ -179,6 +179,24 @@ hadoop-yarn-client-3.1.1.jar hadoop-yarn-common-3.1.1.jar hamcrest-2.1.jar hamcrest-core-1.3.jar +hbase-client-2.2.3.jar +hbase-common-2.2.3.jar +hbase-hadoop-compat-2.2.3.jar +hbase-hadoop2-compat-2.2.3.jar +hbase-http-2.2.3.jar +hbase-mapreduce-2.2.3.jar +hbase-metrics-2.2.3.jar +hbase-metrics-api-2.2.3.jar +hbase-procedure-2.2.3.jar +hbase-protocol-2.2.3.jar +hbase-protocol-shaded-2.2.3.jar +hbase-replication-2.2.3.jar +hbase-server-2.2.3.jar +hbase-shaded-client-byo-hadoop-2.2.3.jar +hbase-shaded-miscellaneous-2.2.1.jar +hbase-shaded-netty-2.2.1.jar +hbase-shaded-protobuf-2.2.1.jar +hbase-zookeeper-2.2.3.jar hibernate-validator-6.0.17.Final.jar hive-classification-3.1.2.jar hive-classification-3.1.3.jar @@ -225,6 +243,7 @@ hk2-utils-3.0.1.jar hppc-0.7.2.jar htrace-core4-4.0.1-incubating.jar htrace-core4-4.1.0-incubating.jar +htrace-core4-4.2.0-incubating.jar httpclient-4.5.13.jar httpclient-4.5.2.jar httpclient-4.5.9.jar @@ -579,21 +598,14 @@ zookeeper-3.4.9.jar zookeeper-3.6.2.jar zookeeper-jute-3.6.2.jar zstd-jni-1.5.0-4.jar -hbase-client-2.2.3.jar -hbase-common-2.2.3.jar -hbase-hadoop-compat-2.2.3.jar -hbase-hadoop2-compat-2.2.3.jar -hbase-http-2.2.3.jar -hbase-mapreduce-2.2.3.jar -hbase-metrics-2.2.3.jar -hbase-metrics-api-2.2.3.jar -hbase-procedure-2.2.3.jar -hbase-protocol-2.2.3.jar -hbase-protocol-shaded-2.2.3.jar -hbase-replication-2.2.3.jar -hbase-server-2.2.3.jar -hbase-shaded-client-byo-hadoop-2.2.3.jar -hbase-shaded-miscellaneous-2.2.1.jar -hbase-shaded-netty-2.2.1.jar -hbase-shaded-protobuf-2.2.1.jar -hbase-zookeeper-2.2.3.jar \ No newline at end of file +disruptor-3.3.6.jar +findbugs-annotations-1.3.9-1.jar +htrace-core4-4.2.0-incubating.jar +jamon-runtime-2.4.1.jar +javax.el-3.0.1-b12.jar +javax.servlet.jsp-2.3.2.jar +javax.servlet.jsp-api-2.3.1.jar +jcodings-1.0.18.jar +joni-2.1.11.jar +libfb303-0.9.3.jar +parquet-hadoop-bundle-1.10.0.jar \ No newline at end of file From 4491db7cfe4bd8e24bc4f90422de9f9ee2fa1572 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 13:28:27 +0800 Subject: [PATCH 13/25] feature: spark-loader bypass server for hbase writing hugegraph-loader fix third-party dependencies error fix #357 --- hugegraph-dist/scripts/dependency/known-dependencies.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/hugegraph-dist/scripts/dependency/known-dependencies.txt b/hugegraph-dist/scripts/dependency/known-dependencies.txt index 6ab318191..68ea1bc24 100644 --- a/hugegraph-dist/scripts/dependency/known-dependencies.txt +++ b/hugegraph-dist/scripts/dependency/known-dependencies.txt @@ -600,7 +600,6 @@ zookeeper-jute-3.6.2.jar zstd-jni-1.5.0-4.jar disruptor-3.3.6.jar findbugs-annotations-1.3.9-1.jar -htrace-core4-4.2.0-incubating.jar jamon-runtime-2.4.1.jar javax.el-3.0.1-b12.jar javax.servlet.jsp-2.3.2.jar From 2bc568b7f595bca4a6ed026e5c5d2abd5a7fe2ae Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 14:19:00 +0800 Subject: [PATCH 14/25] feature: spark-loader bypass server for hbase writing hugegraph-loader relative path fix #357 --- hugegraph-loader/assembly/static/example/spark/struct.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hugegraph-loader/assembly/static/example/spark/struct.json b/hugegraph-loader/assembly/static/example/spark/struct.json index d5c7a94ab..8782a3018 100644 --- a/hugegraph-loader/assembly/static/example/spark/struct.json +++ b/hugegraph-loader/assembly/static/example/spark/struct.json @@ -4,7 +4,7 @@ "label": "person", "input": { "type": "file", - "path": "/Users/alanzhao/Desktop/huge-jacky/incubator-hugegraph-toolchain/hugegraph-loader/assembly/static/example/spark/vertex_person.json", + "path": "example/spark/vertex_person.json", "format": "JSON", "header": ["name", "age", "city"], "charset": "UTF-8", @@ -19,7 +19,7 @@ "label": "software", "input": { "type": "file", - "path": "/Users/alanzhao/Desktop/huge-jacky/incubator-hugegraph-toolchain/hugegraph-loader/assembly/static/example/spark/vertex_software.json", + "path": "example/spark/vertex_software.json", "format": "JSON", "header": ["id","name", "lang", "price","ISBN"], "charset": "GBK" @@ -35,7 +35,7 @@ "target": ["target_name"], "input": { "type": "file", - "path": "/Users/alanzhao/Desktop/huge-jacky/incubator-hugegraph-toolchain/hugegraph-loader/assembly/static/example/spark/edge_knows.json", + "path": "example/spark/edge_knows.json", "format": "JSON", "date_format": "yyyyMMdd", "header": ["source_name","target_name", "date", "weight"] From 286da9768979502b870d4e656ad0db0b7f811186 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 16:37:28 +0800 Subject: [PATCH 15/25] feature: spark-loader bypass server for hbase writing hugegraph-loader fix code style and update dependencies fix #357 --- .../scripts/dependency/known-dependencies.txt | 2 - hugegraph-loader/pom.xml | 30 +++++++------- .../loader/builder/VertexBuilder.java | 4 +- .../loader/direct/util/SinkToHBase.java | 9 +---- .../loader/mapping/ElementMapping.java | 2 +- .../hugegraph/loader/mapping/LoadMapping.java | 16 ++++---- .../loader/spark/HugeGraphSparkLoader.java | 39 +++++++++---------- 7 files changed, 46 insertions(+), 56 deletions(-) diff --git a/hugegraph-dist/scripts/dependency/known-dependencies.txt b/hugegraph-dist/scripts/dependency/known-dependencies.txt index 68ea1bc24..492a9d3dd 100644 --- a/hugegraph-dist/scripts/dependency/known-dependencies.txt +++ b/hugegraph-dist/scripts/dependency/known-dependencies.txt @@ -606,5 +606,3 @@ javax.servlet.jsp-2.3.2.jar javax.servlet.jsp-api-2.3.1.jar jcodings-1.0.18.jar joni-2.1.11.jar -libfb303-0.9.3.jar -parquet-hadoop-bundle-1.10.0.jar \ No newline at end of file diff --git a/hugegraph-loader/pom.xml b/hugegraph-loader/pom.xml index eff34e536..cd2b38ef5 100644 --- a/hugegraph-loader/pom.xml +++ b/hugegraph-loader/pom.xml @@ -386,10 +386,6 @@ org.apache.hadoop hadoop-hdfs - - org.apache.hadoop - hadoop-distcp - org.apache.hadoop hadoop-yarn-registry @@ -431,32 +427,40 @@ jersey-json - org.slf4j - slf4j-api + com.sun.jersey.contribs + jersey-guice - org.slf4j - slf4j-log4j12 + org.apache.parquet + parquet-hadoop-bundle org.apache.hadoop hadoop-common - org.apache.hadoop - hadoop-mapreduce-client-core + org.apache.parquet + parquet-column org.apache.hadoop hadoop-hdfs - com.fasterxml.jackson.core - jackson-annotations + org.apache.thrift + libfb303 com.fasterxml.jackson.core - jackson-core + * + + + org.apache.hadoop + hadoop-distcp + + + org.apache.hadoop + hadoop-mapreduce-client-core diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java index 7f90c6319..89ff06fcc 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java @@ -62,9 +62,8 @@ public List build(String[] names, Object[] values) { @Override public List build(Row row) { - VertexKVPairs kvPairs = this.newKVPairs(this.vertexLabel, - this.mapping.unfold()); + this.mapping.unfold()); String[] names = row.schema().fieldNames(); Object[] values = new Object[row.size()]; for (int i = 0; i < row.size(); i++) { @@ -72,7 +71,6 @@ public List build(Row row) { } kvPairs.extractFromVertex(names, values); return kvPairs.buildVertices(true); - } @Override diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java index 8a437f9c6..28062ab83 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java @@ -163,11 +163,4 @@ public int getPartition(Object key) { } } -} - - - - - - - +} \ No newline at end of file diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/ElementMapping.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/ElementMapping.java index c23413e3d..b5ef9acf7 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/ElementMapping.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/ElementMapping.java @@ -178,7 +178,7 @@ public Object mappingValue(String fieldName, String rawValue) { } public long batchSize() { - return this.batchSize; + return this.batchSize; } public Set selectedFields() { diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java index 0e52f5def..c0b5e7c82 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java @@ -68,10 +68,10 @@ public static LoadMapping of(String filePath) { mapping = MappingUtil.parse(json); } catch (IOException e) { throw new LoadException("Failed to read mapping mapping file '%s'", - e, filePath); + e, filePath); } catch (IllegalArgumentException e) { throw new LoadException("Failed to parse mapping mapping file '%s'", - e, filePath); + e, filePath); } try { mapping.check(); @@ -106,7 +106,7 @@ public void check() throws IllegalArgumentException { "The structs can't be null or empty"); this.structs.forEach(InputStruct::check); Set uniqueIds = this.structs.stream().map(InputStruct::id) - .collect(Collectors.toSet()); + .collect(Collectors.toSet()); E.checkArgument(this.structs.size() == uniqueIds.size(), "The structs cannot contain the same id mapping"); } @@ -136,10 +136,10 @@ public List structsForFailure(LoadOptions options) { String json; try { json = FileUtils.readFileToString(failureFile.headerFile, - charset); + charset); } catch (IOException e) { throw new LoadException("Failed to read header file %s", - failureFile.headerFile); + failureFile.headerFile); } List header = JsonUtil.convertList(json, String.class); source.header(header.toArray(new String[] {})); @@ -172,8 +172,8 @@ private Map groupFailureFiles(File pathDir) { } else { E.checkArgument(Constants.HEADER_SUFFIX.equals(suffix), "The failure data file must end with %s or %s", - Constants.FAILURE_SUFFIX, - Constants.HEADER_SUFFIX); + Constants.FAILURE_SUFFIX, + Constants.HEADER_SUFFIX); failureFile.headerFile = subFile; } failureFiles.put(inputId, failureFile); @@ -188,7 +188,7 @@ public InputStruct struct(String id) { } } throw new IllegalArgumentException(String.format( - "There is no input struct with id '%s'", id)); + "There is no input struct with id '%s'", id)); } private static class FailureFile { diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java index 4a76fd65d..9d2498f8f 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java @@ -65,11 +65,13 @@ import java.util.List; import java.util.Map; import java.util.Properties; + import scala.collection.JavaConverters; public class HugeGraphSparkLoader implements Serializable { public static final Logger LOG = Log.logger(HugeGraphSparkLoader.class); + private final LoadOptions loadOptions; private final Map> builders; @@ -90,7 +92,6 @@ public HugeGraphSparkLoader(String[] args) { } public void load() { - LoadMapping mapping = LoadMapping.of(this.loadOptions.file); this.loadOptions.copyBackendStoreInfo(mapping.getBackendStoreInfo()); List structs = mapping.structs(); @@ -120,9 +121,8 @@ public void load() { LOG.error("spark kryo serialized registration failed"); } SparkSession session = SparkSession.builder() - .config(conf) - .getOrCreate(); - + .config(conf) + .getOrCreate(); SparkContext sc = session.sparkContext(); LongAccumulator totalInsertSuccess = sc.longAccumulator("totalInsertSuccess"); @@ -182,13 +182,12 @@ private LoadContext initPartition( LoadOptions loadOptions, InputStruct struct) { LoadContext context = new LoadContext(loadOptions); for (VertexMapping vertexMapping : struct.vertices()) { - this.builders.put( - new VertexBuilder(context, struct, vertexMapping), - new ArrayList<>()); + this.builders.put(new VertexBuilder(context, struct, vertexMapping), + new ArrayList<>()); } for (EdgeMapping edgeMapping : struct.edges()) { this.builders.put(new EdgeBuilder(context, struct, edgeMapping), - new ArrayList<>()); + new ArrayList<>()); } context.updateSchemaCache(); return context; @@ -208,9 +207,8 @@ private void loadRow(InputStruct struct, Row row, Iterator p, // Insert List graphElements = builderMap.getValue(); if (graphElements.size() >= elementMapping.batchSize() || - (!p.hasNext() && graphElements.size() > 0)) { - flush(builderMap, context.client().graph(), - this.loadOptions.checkVertex); + (!p.hasNext() && graphElements.size() > 0)) { + flush(builderMap, context.client().graph(), this.loadOptions.checkVertex); } } } @@ -224,7 +222,6 @@ private Dataset read(SparkSession ss, InputStruct struct) { case FILE: case HDFS: FileSource fileSource = input.asFileSource(); - String[] header = fileSource.header(); String delimiter = fileSource.delimiter(); String path = fileSource.path(); @@ -247,7 +244,7 @@ private Dataset read(SparkSession ss, InputStruct struct) { break; default: throw new IllegalStateException( - "Unexpected format value: " + format); + "Unexpected format value: " + format); } break; case JDBC: @@ -263,7 +260,7 @@ private Dataset read(SparkSession ss, InputStruct struct) { break; default: throw new AssertionError(String.format( - "Unsupported input source '%s'", input.type())); + "Unsupported input source '%s'", input.type())); } return ds; } @@ -299,8 +296,8 @@ private void parse(Row row, break; default: throw new AssertionError(String.format( - "Unsupported input source '%s'", - struct.input().type())); + "Unsupported input source '%s'", + struct.input().type())); } graphElements.addAll(elements); } @@ -324,15 +321,15 @@ private void flush(Map.Entry> builderMap, BatchVertexRequest.Builder req = new BatchVertexRequest.Builder(); req.vertices((List) (Object) graphElements) - .updatingStrategies(updateStrategyMap) - .createIfNotExist(true); + .updatingStrategies(updateStrategyMap) + .createIfNotExist(true); g.updateVertices(req.build()); } else { BatchEdgeRequest.Builder req = new BatchEdgeRequest.Builder(); req.edges((List) (Object) graphElements) - .updatingStrategies(updateStrategyMap) - .checkVertex(isCheckVertex) - .createIfNotExist(true); + .updatingStrategies(updateStrategyMap) + .checkVertex(isCheckVertex) + .createIfNotExist(true); g.updateEdges(req.build()); } } From 1d5fe80516b66961a99d27ff0ce7d01df75a85fb Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 19:48:31 +0800 Subject: [PATCH 16/25] feature: spark-loader bypass server for hbase writing hugegraph-loader adjust code style fix #357 --- .../com/baidu/hugegraph/loader/mapping/LoadMapping.java | 8 ++++---- .../hugegraph/loader/spark/HugeGraphSparkLoader.java | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java index c0b5e7c82..51280a3e6 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java @@ -98,12 +98,12 @@ public LoadMapping(@JsonProperty("structs") List structs, @Override public void check() throws IllegalArgumentException { E.checkArgument(!StringUtils.isEmpty(this.version), - "The version can't be null or empty"); + "The version can't be null or empty"); E.checkArgument(this.version.equals(Constants.V2_STRUCT_VERSION), - "The version must be '%s', but got '%s'", - Constants.V2_STRUCT_VERSION, this.version); + "The version must be '%s', but got '%s'", + Constants.V2_STRUCT_VERSION, this.version); E.checkArgument(!CollectionUtils.isEmpty(this.structs), - "The structs can't be null or empty"); + "The structs can't be null or empty"); this.structs.forEach(InputStruct::check); Set uniqueIds = this.structs.stream().map(InputStruct::id) .collect(Collectors.toSet()); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java index 9d2498f8f..984a9fb9e 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java @@ -93,9 +93,11 @@ public HugeGraphSparkLoader(String[] args) { public void load() { LoadMapping mapping = LoadMapping.of(this.loadOptions.file); - this.loadOptions.copyBackendStoreInfo(mapping.getBackendStoreInfo()); List structs = mapping.structs(); boolean sinkType = this.loadOptions.sinkType; + if(!sinkType){ + this.loadOptions.copyBackendStoreInfo(mapping.getBackendStoreInfo()); + } SparkConf conf = new SparkConf() .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")// kryo序列化 .set("spark.kryo.registrationRequired", "true"); @@ -321,8 +323,8 @@ private void flush(Map.Entry> builderMap, BatchVertexRequest.Builder req = new BatchVertexRequest.Builder(); req.vertices((List) (Object) graphElements) - .updatingStrategies(updateStrategyMap) - .createIfNotExist(true); + .updatingStrategies(updateStrategyMap) + .createIfNotExist(true); g.updateVertices(req.build()); } else { BatchEdgeRequest.Builder req = new BatchEdgeRequest.Builder(); From b5a9a2eb440ae099182fcfc3c4b19b9993d915cf Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Tue, 8 Nov 2022 22:31:49 +0800 Subject: [PATCH 17/25] feature: spark-loader bypass server for hbase writing hugegraph-loader adjust code style fix #357 --- .../assembly/static/example/spark/struct.json | 7 +++---- .../static/example/spark/vertex_person.json | 3 +-- .../static/example/spark/vertex_software.json | 3 +-- .../hugegraph/loader/builder/EdgeBuilder.java | 8 ++++---- .../hugegraph/loader/constant/Constants.java | 2 ++ .../loader/direct/loader/DirectLoader.java | 3 ++- .../direct/loader/HBaseDirectLoader.java | 18 ++++++++---------- .../loader/direct/util/SinkToHBase.java | 7 +++---- .../hugegraph/loader/executor/LoadOptions.java | 5 ++--- .../hugegraph/loader/mapping/LoadMapping.java | 8 ++++---- .../loader/metrics/LoadDistributeMetrics.java | 6 ------ .../loader/spark/HugeGraphSparkLoader.java | 12 +++++------- 12 files changed, 35 insertions(+), 47 deletions(-) diff --git a/hugegraph-loader/assembly/static/example/spark/struct.json b/hugegraph-loader/assembly/static/example/spark/struct.json index 8782a3018..275bdbd32 100644 --- a/hugegraph-loader/assembly/static/example/spark/struct.json +++ b/hugegraph-loader/assembly/static/example/spark/struct.json @@ -50,9 +50,8 @@ { "edge_tablename": "hugegraph:g_oe", "vertex_tablename": "hugegraph:g_v", - "hbase_zookeeper_quorum":"127.0.0.1", - "hbase_zookeeper_property_clientPort":"2181", - "zookeeper_znode_parent":"/hbase" - + "hbase_zookeeper_quorum": "127.0.0.1", + "hbase_zookeeper_property_clientPort": "2181", + "zookeeper_znode_parent": "/hbase" } } diff --git a/hugegraph-loader/assembly/static/example/spark/vertex_person.json b/hugegraph-loader/assembly/static/example/spark/vertex_person.json index 76fa1dd80..b18e0d4e3 100644 --- a/hugegraph-loader/assembly/static/example/spark/vertex_person.json +++ b/hugegraph-loader/assembly/static/example/spark/vertex_person.json @@ -3,5 +3,4 @@ {"name": "josh", "age": "32", "city": "Beijing"} {"name": "peter", "age": "35", "city": "Shanghai"} {"name": "li,nary", "age": "26", "city": "Wu,han"} -{"name": "tom", "age": "null", "city": "NULL"} - +{"name": "tom", "age": "null", "city": "NULL"} \ No newline at end of file diff --git a/hugegraph-loader/assembly/static/example/spark/vertex_software.json b/hugegraph-loader/assembly/static/example/spark/vertex_software.json index ba68ffe8b..bffb1ca24 100644 --- a/hugegraph-loader/assembly/static/example/spark/vertex_software.json +++ b/hugegraph-loader/assembly/static/example/spark/vertex_software.json @@ -1,3 +1,2 @@ { "name": "lop", "lang": "java","price": "328","ISBN": "ISBN978-7-107-18618-5"} -{ "name": "ripple", "lang": "java","price": "199","ISBN": "ISBN978-7-100-13678-5"} - +{ "name": "ripple", "lang": "java","price": "199","ISBN": "ISBN978-7-100-13678-5"} \ No newline at end of file diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java index 8c8f79dde..f7fae075f 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java @@ -69,7 +69,7 @@ public EdgeBuilder(LoadContext context, InputStruct struct, public EdgeMapping mapping() { return this.mapping; } - + @Override public List build(String[] names, Object[] values) { if (this.vertexIdsIndex == null || @@ -138,15 +138,15 @@ public List build(Row row) { } E.checkArgument(sources.size() == 1 || targets.size() == 1 || sources.size() == targets.size(), - "The elements number of source and target must be: " + + "The elements number of source and target must be: " + "1 to n, n to 1, n to n"); int size = Math.max(sources.size(), targets.size()); List edges = new ArrayList<>(size); for (int i = 0; i < size; i++) { Vertex source = i < sources.size() ? - sources.get(i) : sources.get(0); + sources.get(i) : sources.get(0); Vertex target = i < targets.size() ? - targets.get(i) : targets.get(0); + targets.get(i) : targets.get(0); Edge edge = new Edge(this.mapping.label()); edge.source(source); edge.target(target); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java index fb42312ae..d15e6e7e5 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java @@ -83,4 +83,6 @@ public final class Constants { public static final String LOAD_DATA_PARSE_SUFFIX = "parse"; public static final String LOAD_DATA_SER_SUFFIX = "ser"; public static final String LOAD_DATA_INSERT_SUFFIX = "insert"; + public static final String BYPASS_HFILE_PATH_ = "insert"; + ///hfile-gen } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java index bbbc4359d..c197f90bc 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java @@ -36,6 +36,7 @@ import java.util.List; public abstract class DirectLoader implements Serializable { + LoadOptions loadOptions; InputStruct struct; @@ -45,7 +46,7 @@ public DirectLoader(LoadOptions loadOptions, this.struct = struct; } - public final void bulkload(Dataset ds) { + public final void bulkload(Dataset ds) { JavaPairRDD javaPairRDD = buildVertexAndEdge(ds); String path = generateFiles(javaPairRDD); loadFiles(path); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java index 89045ed5f..dab182842 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java @@ -94,10 +94,10 @@ public JavaPairRDD buildVertexAndEdge(Dataset< LOG.info("buildAndSer start execute >>>>"); JavaPairRDD tuple2KeyValueJavaPairRDD = ds.toJavaRDD().mapPartitionsToPair( - new PairFlatMapFunction, ImmutableBytesWritable, KeyValue>() { - @Override + new PairFlatMapFunction, ImmutableBytesWritable, KeyValue>() { + @Override public Iterator> call( - Iterator rowIterator) throws Exception { + Iterator rowIterator) throws Exception { HBaseSerializer serializer = new HBaseSerializer( HugeClientHolder.create(loadOptions), @@ -113,8 +113,8 @@ public Iterator> call( } serializer.close(); return result.iterator(); - } - } + } + } ); return tuple2KeyValueJavaPairRDD; } @@ -143,11 +143,11 @@ String generateFiles(JavaPairRDD buildAndSerRd HFileOutputFormat2.class, conf ); - LOG.info("Saved to HFiles to: " + path); + LOG.info("Saved HFiles to: '{}'", path); flushPermission(conf,path); return path; } catch (IOException e) { - e.printStackTrace(); + LOG.error("generateFiles failed",e); } return Constants.EMPTY_STR; @@ -171,7 +171,7 @@ public void loadFiles(String path) { try { sinkToHBase.loadHfiles(path, getTableName());// BulkLoad HFile to HBase } catch (Exception e) { - e.printStackTrace(); + LOG.error("hfile failed to be loaded",e); } } @@ -207,9 +207,7 @@ List> buildAndSer (HBaseSerializer ser switch (struct.input().type()) { case FILE: case HDFS: - elementsElement = builder.build(row); - break; default: throw new AssertionError(String.format( diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java index 28062ab83..95040ca69 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java @@ -69,7 +69,7 @@ private Optional getConnection() { try { conn = ConnectionFactory.createConnection(baseConf.get()); } catch (IOException e) { - e.printStackTrace(); + LOG.error("get hbase connection failed",e); } return Optional.ofNullable(conn); } @@ -144,8 +144,8 @@ public int getPartition(Object key) { String keyString = Bytes.toString(immutableBytesWritableKey.get()); for (List range : rangeMap.keySet()) { if (keyString.compareToIgnoreCase(range.get(0)) >= 0 && - ((keyString.compareToIgnoreCase(range.get(1)) < 0) || - range.get(1).equals(""))) { + ((keyString.compareToIgnoreCase(range.get(1)) < 0) || + range.get(1).equals(""))) { return rangeMap.get(range); } } @@ -162,5 +162,4 @@ public int getPartition(Object key) { } } } - } \ No newline at end of file diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java index aa7196220..edc9a6ab7 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java @@ -45,7 +45,6 @@ public class LoadOptions implements Serializable { public static final String HTTP_SCHEMA = "http"; private static final int CPUS = Runtime.getRuntime().availableProcessors(); - @Parameter(names = {"-f", "--file"}, required = true, arity = 1, validateWith = {FileValidator.class}, description = "The path of the data mapping description file") @@ -368,8 +367,8 @@ public void validate(String name, String value) { int retry = Integer.parseInt(value); if (retry <= 0) { throw new ParameterException(String.format( - "Parameter '%s' should be positive, but got '%s'", - name, value)); + "Parameter '%s' should be positive, but got '%s'", + name, value)); } } } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java index 51280a3e6..0bd795582 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java @@ -108,7 +108,7 @@ public void check() throws IllegalArgumentException { Set uniqueIds = this.structs.stream().map(InputStruct::id) .collect(Collectors.toSet()); E.checkArgument(this.structs.size() == uniqueIds.size(), - "The structs cannot contain the same id mapping"); + "The structs cannot contain the same id mapping"); } public List structs() { @@ -136,7 +136,7 @@ public List structsForFailure(LoadOptions options) { String json; try { json = FileUtils.readFileToString(failureFile.headerFile, - charset); + charset); } catch (IOException e) { throw new LoadException("Failed to read header file %s", failureFile.headerFile); @@ -157,7 +157,7 @@ public List structsForFailure(LoadOptions options) { private Map groupFailureFiles(File pathDir) { File[] subFiles = pathDir.listFiles(); E.checkArgument(subFiles != null && subFiles.length >= 1, - "Every input struct should have a failure data file, " + + "Every input struct should have a failure data file, " + "and a header file if need it"); Map failureFiles = new LinkedHashMap<>(); for (File subFile : subFiles) { @@ -171,7 +171,7 @@ private Map groupFailureFiles(File pathDir) { failureFile.dataFile = subFile; } else { E.checkArgument(Constants.HEADER_SUFFIX.equals(suffix), - "The failure data file must end with %s or %s", + "The failure data file must end with %s or %s", Constants.FAILURE_SUFFIX, Constants.HEADER_SUFFIX); failureFile.headerFile = subFile; diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java index 1f8690511..3345cd169 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/metrics/LoadDistributeMetrics.java @@ -57,7 +57,6 @@ public void init(SparkContext sc) { Constants.UNDERLINE_STR + Constants.LOAD_DATA_INSERT_SUFFIX); metrics.parseSuccess = sc.longAccumulator(mapping.label() + Constants.UNDERLINE_STR + Constants.LOAD_DATA_PARSE_SUFFIX); - } for (EdgeMapping mapping : this.struct.edges()) { Metrics metrics = this.edgeDisMetrics.get(mapping.label()); @@ -65,12 +64,10 @@ public void init(SparkContext sc) { Constants.UNDERLINE_STR + Constants.LOAD_DATA_INSERT_SUFFIX); metrics.parseSuccess = sc.longAccumulator(mapping.label() + Constants.UNDERLINE_STR + Constants.LOAD_DATA_PARSE_SUFFIX); - } } public void increaseDisVertexParseSuccess(ElementMapping mapping) { - this.disMetrics(mapping).parseSuccess.add(1); } @@ -87,7 +84,6 @@ public void plusDisVertexInsertSuccess(ElementMapping mapping, Long count) { } public void increaseDisEdgeParseSuccess(ElementMapping mapping) { - this.disMetrics(mapping).parseSuccess.add(1); } @@ -96,7 +92,6 @@ public void pluseDisEdgeParseSuccess(ElementMapping mapping, Long count) { } public void increaseDisEdgeInsertSuccess(ElementMapping mapping) { - this.disMetrics(mapping).insertSuccess.add(1); } @@ -138,7 +133,6 @@ public static class Metrics implements Serializable { private LongAccumulator insertFailure; public Metrics() { - } public long parseSuccess() { diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java index 984a9fb9e..4502aa929 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java @@ -102,7 +102,7 @@ public void load() { .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")// kryo序列化 .set("spark.kryo.registrationRequired", "true"); try { - conf.registerKryoClasses ( + conf.registerKryoClasses( new Class[] { org.apache.hadoop.hbase.io.ImmutableBytesWritable.class, @@ -129,12 +129,12 @@ public void load() { LongAccumulator totalInsertSuccess = sc.longAccumulator("totalInsertSuccess"); for (InputStruct struct : structs) { - LOG.info("\n init" + struct.input().asFileSource().path() + - " distribute metrics---- \n"); + LOG.info("\n init {} distribute metrics---- \n", + struct.input().asFileSource().path()); LoadDistributeMetrics loadDistributeMetrics = new LoadDistributeMetrics(struct); loadDistributeMetrics.init(sc); - LOG.info("\n load dat info: \t" + struct.input().asFileSource().path() + - "\n start load data ; \n"); + LOG.info("\n load dat info: \t {} \n start load data ; \n", + struct.input().asFileSource().path()); Dataset ds = read(session, struct); if (sinkType) { LOG.info("\n ------ spark api start load data ------ \n"); @@ -167,12 +167,10 @@ public void load() { sc.stop(); session.close(); session.stop(); - } private void collectLoadMetrics (LoadDistributeMetrics loadMetrics, LongAccumulator totalInsertSuccess) { - Long edgeInsertSuccess = loadMetrics.readEdgeInsertSuccess(); Long vertexInsertSuccess = loadMetrics.readVertexInsertSuccess(); totalInsertSuccess.add(edgeInsertSuccess); From 32035f7f7e126a9b28802a0556252672546f9412 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Wed, 9 Nov 2022 00:58:59 +0800 Subject: [PATCH 18/25] feature: spark-loader bypass server for hbase writing hugegraph-loader adjust code style fix #357 --- .../static/example/spark/schema.groovy | 2 -- .../static/example/spark/vertex_person.json | 2 +- .../static/example/spark/vertex_software.json | 2 +- .../hugegraph/loader/constant/Constants.java | 2 -- .../direct/loader/HBaseDirectLoader.java | 30 +++++++++---------- .../loader/direct/util/SinkToHBase.java | 3 +- .../loader/executor/LoadOptions.java | 5 ++-- .../loader/mapping/BackendStoreInfo.java | 3 +- .../loader/spark/HugeGraphSparkLoader.java | 23 +++++++------- 9 files changed, 33 insertions(+), 39 deletions(-) diff --git a/hugegraph-loader/assembly/static/example/spark/schema.groovy b/hugegraph-loader/assembly/static/example/spark/schema.groovy index ea69bb530..a1d68ac46 100644 --- a/hugegraph-loader/assembly/static/example/spark/schema.groovy +++ b/hugegraph-loader/assembly/static/example/spark/schema.groovy @@ -32,5 +32,3 @@ schema.edgeLabel("created") .properties("date", "weight") .ifNotExist() .create(); - - diff --git a/hugegraph-loader/assembly/static/example/spark/vertex_person.json b/hugegraph-loader/assembly/static/example/spark/vertex_person.json index b18e0d4e3..e018df411 100644 --- a/hugegraph-loader/assembly/static/example/spark/vertex_person.json +++ b/hugegraph-loader/assembly/static/example/spark/vertex_person.json @@ -3,4 +3,4 @@ {"name": "josh", "age": "32", "city": "Beijing"} {"name": "peter", "age": "35", "city": "Shanghai"} {"name": "li,nary", "age": "26", "city": "Wu,han"} -{"name": "tom", "age": "null", "city": "NULL"} \ No newline at end of file +{"name": "tom", "age": "null", "city": "NULL"} diff --git a/hugegraph-loader/assembly/static/example/spark/vertex_software.json b/hugegraph-loader/assembly/static/example/spark/vertex_software.json index bffb1ca24..cd9dbf3cb 100644 --- a/hugegraph-loader/assembly/static/example/spark/vertex_software.json +++ b/hugegraph-loader/assembly/static/example/spark/vertex_software.json @@ -1,2 +1,2 @@ { "name": "lop", "lang": "java","price": "328","ISBN": "ISBN978-7-107-18618-5"} -{ "name": "ripple", "lang": "java","price": "199","ISBN": "ISBN978-7-100-13678-5"} \ No newline at end of file +{ "name": "ripple", "lang": "java","price": "199","ISBN": "ISBN978-7-100-13678-5"} diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java index d15e6e7e5..fb42312ae 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/constant/Constants.java @@ -83,6 +83,4 @@ public final class Constants { public static final String LOAD_DATA_PARSE_SUFFIX = "parse"; public static final String LOAD_DATA_SER_SUFFIX = "ser"; public static final String LOAD_DATA_INSERT_SUFFIX = "insert"; - public static final String BYPASS_HFILE_PATH_ = "insert"; - ///hfile-gen } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java index dab182842..41cd21307 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java @@ -85,13 +85,13 @@ public String getTableName() { } public Integer getTablePartitions () { - return struct.edges().size() > 0 ? - loadOptions.edgePartitions : - loadOptions.vertexPartitions; + return struct.edges().size() > 0 ? + loadOptions.edgePartitions : + loadOptions.vertexPartitions; } public JavaPairRDD buildVertexAndEdge(Dataset ds) { - LOG.info("buildAndSer start execute >>>>"); + LOG.info("Start build vertexs and edges"); JavaPairRDD tuple2KeyValueJavaPairRDD = ds.toJavaRDD().mapPartitionsToPair( new PairFlatMapFunction, ImmutableBytesWritable, KeyValue>() { @@ -111,7 +111,7 @@ public Iterator> call( buildAndSer(serializer, row,buildersForGraphElement); result.addAll(serList); } - serializer.close(); + serializer.close(); return result.iterator(); } } @@ -121,7 +121,7 @@ public Iterator> call( @Override String generateFiles(JavaPairRDD buildAndSerRdd) { - LOG.info("bulkload start execute>>>"); + LOG.info("Start to generate hfile"); try { Tuple2 tuple = sinkToHBase.getPartitionerByTableName(getTablePartitions(), getTableName()); @@ -147,11 +147,9 @@ String generateFiles(JavaPairRDD buildAndSerRd flushPermission(conf,path); return path; } catch (IOException e) { - LOG.error("generateFiles failed",e); + LOG.error("Failed to generate files", e); } - return Constants.EMPTY_STR; - } public String getHFilePath (Configuration conf) throws IOException { @@ -160,7 +158,7 @@ public String getHFilePath (Configuration conf) throws IOException { String pathStr = fs.getWorkingDirectory().toString() + "/hfile-gen" + "/" + timeStr + "/"; Path hfileGenPath = new Path(pathStr); if (fs.exists(hfileGenPath)) { - LOG.info("\n delete hfile path \n"); + LOG.info("\n Delete the path where the hfile is generated,path {} ", pathStr); fs.delete(hfileGenPath,true); } return pathStr; @@ -171,14 +169,14 @@ public void loadFiles(String path) { try { sinkToHBase.loadHfiles(path, getTableName());// BulkLoad HFile to HBase } catch (Exception e) { - LOG.error("hfile failed to be loaded",e); + LOG.error(" Failed to load hfiles", e); } } private void flushPermission (Configuration conf, String path) { FsShell shell = new FsShell(conf); try { - LOG.info("shell start execute"); + LOG.info("Chmod hfile directory permission"); shell.run(new String[]{"-chmod", "-R", "777", path}); shell.close(); } catch (Exception e) { @@ -218,7 +216,7 @@ List> buildAndSer (HBaseSerializer ser boolean isVertex = builder.mapping().type().isVertex(); if (isVertex) { for (Vertex vertex : (List) (Object) elementsElement) { - LOG.debug("vertex already build done " + vertex.toString()); + LOG.debug("vertex already build done {} ", vertex.toString()); Tuple2 tuple2 = vertexSerialize(serializer,vertex); loadDistributeMetrics.increaseDisVertexInsertSuccess(builder.mapping()); @@ -226,7 +224,7 @@ List> buildAndSer (HBaseSerializer ser } } else { for (Edge edge : (List) (Object) elementsElement) { - LOG.debug("edge already build done " + edge.toString()); + LOG.debug("edge already build done {}", edge.toString()); Tuple2 tuple2 = edgeSerialize(serializer,edge); loadDistributeMetrics.increaseDisEdgeInsertSuccess(builder.mapping()); @@ -240,7 +238,7 @@ List> buildAndSer (HBaseSerializer ser private Tuple2 edgeSerialize (HBaseSerializer serializer, Edge edge) { - LOG.debug("edge start serialize " + edge.toString()); + LOG.debug("edge start serialize {}", edge.toString()); byte[] rowkey = serializer.getKeyBytes(edge); byte[] values = serializer.getValueBytes(edge); ImmutableBytesWritable rowKey = new ImmutableBytesWritable(); @@ -254,7 +252,7 @@ private Tuple2 edgeSerialize (HBaseSerializer private Tuple2 vertexSerialize (HBaseSerializer serializer, Vertex vertex) { - LOG.debug("vertex start serialize " + vertex.toString()); + LOG.debug("vertex start serialize {}", vertex.toString()); byte[] rowkey = serializer.getKeyBytes(vertex); byte[] values = serializer.getValueBytes(vertex); ImmutableBytesWritable rowKey = new ImmutableBytesWritable(); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java index 95040ca69..3184d84e7 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java @@ -96,10 +96,10 @@ public void loadHfiles (String path, String tableName) throws Exception { bulkLoadHFilesTool.bulkLoad(table.getName(), new Path(path)); table.close(); conn.close(); - } public class IntPartitioner extends Partitioner { + private final int numPartitions; public Map, Integer> rangeMap = new HashMap<>(); private String tableName; @@ -162,4 +162,5 @@ public int getPartition(Object key) { } } } + } \ No newline at end of file diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java index edc9a6ab7..2178a5088 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java @@ -300,7 +300,6 @@ public void copyBackendStoreInfo (BackendStoreInfo backendStoreInfo) { this.hbaseZKParent = backendStoreInfo.getHbaseZKParent(); this.hbaseZKPort = backendStoreInfo.getHbaseZKPort(); this.hbaseZKQuorum = backendStoreInfo.getHbaseZKQuorum(); - } public static class UrlValidator implements IParameterValidator { @@ -341,8 +340,8 @@ public void validate(String name, String value) { File file = new File(value); if (!file.exists() || !file.isDirectory()) { throw new ParameterException(String.format( - "Ensure the directory exists and is indeed a " + - "directory instead of a file: '%s'", value)); + "Ensure the directory exists and is indeed a " + + "directory instead of a file: '%s'", value)); } } } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java index 638ca011d..1b533262d 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/BackendStoreInfo.java @@ -23,8 +23,9 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; @JsonPropertyOrder({"edge_tablename", "vertex_tablename", "hbase_zookeeper_quorum", - "hbase_zookeeper_property_clientPort", "zookeeper_znode_parent"}) + "hbase_zookeeper_property_clientPort", "zookeeper_znode_parent"}) public class BackendStoreInfo { + @JsonProperty("edge_tablename") private String edgeTablename; @JsonProperty("vertex_tablename") diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java index 4502aa929..34445aae2 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java @@ -129,15 +129,15 @@ public void load() { LongAccumulator totalInsertSuccess = sc.longAccumulator("totalInsertSuccess"); for (InputStruct struct : structs) { - LOG.info("\n init {} distribute metrics---- \n", + LOG.info("\n Initializes the accumulator corresponding to the {} ", struct.input().asFileSource().path()); LoadDistributeMetrics loadDistributeMetrics = new LoadDistributeMetrics(struct); loadDistributeMetrics.init(sc); - LOG.info("\n load dat info: \t {} \n start load data ; \n", + LOG.info("\n Start to load data, data info is: \t {} ", struct.input().asFileSource().path()); Dataset ds = read(session, struct); if (sinkType) { - LOG.info("\n ------ spark api start load data ------ \n"); + LOG.info("\n Start to load data using spark apis \n"); ds.foreachPartition((Iterator p) -> { LoadContext context = initPartition(this.loadOptions, struct); p.forEachRemaining((Row row) -> { @@ -147,7 +147,7 @@ public void load() { }); } else { - LOG.info("\n spark bulkload gen hfile start \n"); + LOG.info("\n Start to load data using spark bulkload \n"); // gen-hfile HBaseDirectLoader directLoader = new HBaseDirectLoader(loadOptions, struct,loadDistributeMetrics); @@ -155,27 +155,26 @@ public void load() { } collectLoadMetrics(loadDistributeMetrics,totalInsertSuccess); - LOG.info(" \n load data info : \t" + struct.input().asFileSource().path() + - "\n load data finish!!!; \n"); + LOG.info(" \n Finished load {} data ", + struct.input().asFileSource().path()); } Long totalInsertSuccessCnt = totalInsertSuccess.value(); - LOG.info("\n ------------The data import task is complete-------------------\n" + - "\n insertSuccess cnt:\t" + totalInsertSuccess + " \n" + + LOG.info("\n ------------The data load task is complete-------------------\n" + + "\n insertSuccesscnt:\t {}" + "\n ---------------------------------------------\n" - ); + , totalInsertSuccessCnt); sc.stop(); session.close(); session.stop(); } - private void collectLoadMetrics (LoadDistributeMetrics loadMetrics, - LongAccumulator totalInsertSuccess) { + private void collectLoadMetrics(LoadDistributeMetrics loadMetrics, + LongAccumulator totalInsertSuccess) { Long edgeInsertSuccess = loadMetrics.readEdgeInsertSuccess(); Long vertexInsertSuccess = loadMetrics.readVertexInsertSuccess(); totalInsertSuccess.add(edgeInsertSuccess); totalInsertSuccess.add(vertexInsertSuccess); - } private LoadContext initPartition( From 662901942a9a236b0ff68171e11d9a64982b6126 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Wed, 9 Nov 2022 01:07:18 +0800 Subject: [PATCH 19/25] feature: spark-loader bypass server for hbase writing hugegraph-loader adjust code style fix #357 --- .../com/baidu/hugegraph/loader/direct/util/SinkToHBase.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java index 3184d84e7..6e04aa85d 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java @@ -162,5 +162,4 @@ public int getPartition(Object key) { } } } - -} \ No newline at end of file +} From b31b355e386c5098a0cd819d7946a03f6b8ff78d Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Wed, 9 Nov 2022 09:42:47 +0800 Subject: [PATCH 20/25] feature: spark-loader bypass server for hbase writing hugegraph-loader adjust code style fix #357 --- .../static/example/spark/schema.groovy | 1 - .../direct/loader/HBaseDirectLoader.java | 2 +- .../loader/executor/LoadOptions.java | 153 +++++++++--------- 3 files changed, 77 insertions(+), 79 deletions(-) diff --git a/hugegraph-loader/assembly/static/example/spark/schema.groovy b/hugegraph-loader/assembly/static/example/spark/schema.groovy index a1d68ac46..15e829d31 100644 --- a/hugegraph-loader/assembly/static/example/spark/schema.groovy +++ b/hugegraph-loader/assembly/static/example/spark/schema.groovy @@ -19,7 +19,6 @@ schema.vertexLabel("software") .ifNotExist() .create(); - schema.edgeLabel("knows") .sourceLabel("person") .targetLabel("person") diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java index 41cd21307..162b82d75 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java @@ -111,7 +111,7 @@ public Iterator> call( buildAndSer(serializer, row,buildersForGraphElement); result.addAll(serList); } - serializer.close(); + serializer.close(); return result.iterator(); } } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java index 2178a5088..7ba19e9d8 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java @@ -46,194 +46,193 @@ public class LoadOptions implements Serializable { private static final int CPUS = Runtime.getRuntime().availableProcessors(); @Parameter(names = {"-f", "--file"}, required = true, arity = 1, - validateWith = {FileValidator.class}, - description = "The path of the data mapping description file") + validateWith = {FileValidator.class}, + description = "The path of the data mapping description file") public String file; @Parameter(names = {"-s", "--schema"}, arity = 1, - validateWith = {FileValidator.class}, - description = "The schema file path which to create manually") + validateWith = {FileValidator.class}, + description = "The schema file path which to create manually") public String schema; @Parameter(names = {"-g", "--graph"}, required = true, arity = 1, - description = "The namespace of the graph to load into") + description = "The namespace of the graph to load into") public String graph; @Parameter(names = {"-h", "--host"}, arity = 1, - validateWith = {UrlValidator.class}, - description = "The host/IP of HugeGraphServer") + validateWith = {UrlValidator.class}, + description = "The host/IP of HugeGraphServer") public String host = "localhost"; @Parameter(names = {"-p", "--port"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The port of HugeGraphServer") + validateWith = {PositiveValidator.class}, + description = "The port of HugeGraphServer") public int port = 8080; @Parameter(names = {"--username"}, arity = 1, - description = "The username of graph for authentication") + description = "The username of graph for authentication") public String username = null; @Parameter(names = {"--protocol"}, arity = 1, - validateWith = {ProtocolValidator.class}, - description = "The protocol of HugeGraphServer, " + + validateWith = {ProtocolValidator.class}, + description = "The protocol of HugeGraphServer, " + "allowed values are: http or https") public String protocol = "http"; @Parameter(names = {"--trust-store-file"}, arity = 1, - description = "The path of client truststore file used " + + description = "The path of client truststore file used " + "when https protocol is enabled") public String trustStoreFile = null; @Parameter(names = {"--trust-store-password"}, arity = 1, - description = "The password of client truststore file used " + + description = "The password of client truststore file used " + "when https protocol is enabled") public String trustStoreToken = null; @Parameter(names = {"--token"}, arity = 1, - description = "The token of graph for authentication") + description = "The token of graph for authentication") public String token = null; @Parameter(names = {"--clear-all-data"}, arity = 1, - description = "Whether to clear all old data before loading") + description = "Whether to clear all old data before loading") public boolean clearAllData = false; @Parameter(names = {"--clear-timeout"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The timeout waiting for clearing all data ") + validateWith = {PositiveValidator.class}, + description = "The timeout waiting for clearing all data ") public int clearTimeout = 240; @Parameter(names = {"--incremental-mode"}, arity = 1, - description = "Load data from the breakpoint of last time") + description = "Load data from the breakpoint of last time") public boolean incrementalMode = false; @Parameter(names = {"--failure-mode"}, arity = 1, - description = "Load data from the failure records, in this " + - "mode, only full load is supported, any read " + - "or parsing errors will cause load task stop") + description = "Load data from the failure records, in this " + + "mode, only full load is supported, any read " + + "or parsing errors will cause load task stop") public boolean failureMode = false; @Parameter(names = {"--batch-insert-threads"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The number of threads to execute batch insert") + validateWith = {PositiveValidator.class}, + description = "The number of threads to execute batch insert") public int batchInsertThreads = CPUS; @Parameter(names = {"--single-insert-threads"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The number of threads to execute single insert") + validateWith = {PositiveValidator.class}, + description = "The number of threads to execute single insert") public int singleInsertThreads = 8; @Parameter(names = {"--max-conn"}, arity = 1, - description = "Max number of HTTP connections to server") + description = "Max number of HTTP connections to server") public int maxConnections = CPUS * 4; @Parameter(names = {"--max-conn-per-route"}, arity = 1, - description = "Max number of HTTP connections to each route") + description = "Max number of HTTP connections to each route") public int maxConnectionsPerRoute = CPUS * 2; @Parameter(names = {"--batch-size"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The number of lines in each submit") + validateWith = {PositiveValidator.class}, + description = "The number of lines in each submit") public int batchSize = 500; @Parameter(names = {"--cdc-flush-interval"}, arity = 1, - description = "The flush interval for flink cdc") + description = "The flush interval for flink cdc") public int flushIntervalMs = 30000; @Parameter(names = {"--cdc-sink-parallelism"}, arity = 1, - description = "The sink parallelism for flink cdc") + description = "The sink parallelism for flink cdc") public int sinkParallelism = 1; @Parameter(names = {"--shutdown-timeout"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The timeout of awaitTermination in seconds") + validateWith = {PositiveValidator.class}, + description = "The timeout of awaitTermination in seconds") public int shutdownTimeout = 10; @Parameter(names = {"--check-vertex"}, arity = 1, - description = "Check vertices exists while inserting edges") + description = "Check vertices exists while inserting edges") public boolean checkVertex = false; @Parameter(names = {"--max-read-errors"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The maximum number of lines that read error " + + validateWith = {PositiveValidator.class}, + description = "The maximum number of lines that read error " + "before exiting") public int maxReadErrors = 1; @Parameter(names = {"--max-parse-errors"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The maximum number of lines that parse error " + + validateWith = {PositiveValidator.class}, + description = "The maximum number of lines that parse error " + "before exiting") public int maxParseErrors = 1; @Parameter(names = {"--max-insert-errors"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The maximum number of lines that insert error " + + validateWith = {PositiveValidator.class}, + description = "The maximum number of lines that insert error " + "before exiting") public int maxInsertErrors = 500; @Parameter(names = {"--timeout"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The timeout of HugeClient request") + validateWith = {PositiveValidator.class}, + description = "The timeout of HugeClient request") public int timeout = 60; @Parameter(names = {"--retry-times"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "Setting the max retry times when loading timeout") + validateWith = {PositiveValidator.class}, + description = "Setting the max retry times when loading timeout") public int retryTimes = 3; @Parameter(names = {"--retry-interval"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "Setting the interval time before retrying") + validateWith = {PositiveValidator.class}, + description = "Setting the interval time before retrying") public int retryInterval = 10; @Parameter(names = {"--max-read-lines"}, arity = 1, - validateWith = {PositiveValidator.class}, - description = "The maximum number of read lines, when reached " + + validateWith = {PositiveValidator.class}, + description = "The maximum number of read lines, when reached " + "this number, the load task will stop") public long maxReadLines = -1L; @Parameter(names = {"--dry-run"}, arity = 1, - description = "Dry run means that only parse but doesn't load") + description = "Dry run means that only parse but doesn't load") public boolean dryRun = false; @Parameter(names = {"--print-progress"}, arity = 1, - description = "Whether to print real-time load progress") + description = "Whether to print real-time load progress") public boolean printProgress = true; @Parameter(names = {"--test-mode"}, arity = 1, - description = "Whether the hugegraph-loader work in test mode") + description = "Whether the hugegraph-loader work in test mode") public boolean testMode = false; @Parameter(names = {"--help"}, help = true, - description = "Print usage of HugeGraphLoader") + description = "Print usage of HugeGraphLoader") public boolean help; @Parameter(names = {"--sink-type"}, arity = 1, - description = "Sink to different storage") + description = "Sink to different storage") public boolean sinkType = true; - @Parameter(names = {"--edge-partitions"}, arity = 1, - description = "The number of partitions of the HBase edge table") + description = "The number of partitions of the HBase edge table") public int edgePartitions = 64; @Parameter(names = {"--vertex-partitions"}, arity = 1, - description = "The number of partitions of the HBase vertex table") + description = "The number of partitions of the HBase vertex table") public int vertexPartitions = 64; @Parameter(names = {"edgeTablename"}, arity = 1, - description = "edgeTablename") + description = "edgeTablename") public String edgeTablename; @Parameter(names = {"vertexTablename"}, arity = 1, - description = "vertexTablename") + description = "vertexTablename") public String vertexTablename; @Parameter(names = {"hbaseZKQuorum"}, arity = 1, - description = "hbaseZKQuorum") + description = "hbaseZKQuorum") public String hbaseZKQuorum; @Parameter(names = {"hbaseZKPort"}, arity = 1, - description = "hbaseZKPort") + description = "hbaseZKPort") public String hbaseZKPort; @Parameter(names = {"hbaseZKParent"}, arity = 1, - description = "hbaseZKParent") + description = "hbaseZKParent") public String hbaseZKParent; public String workModeString() { @@ -249,8 +248,8 @@ public String workModeString() { public static LoadOptions parseOptions(String[] args) { LoadOptions options = new LoadOptions(); JCommander commander = JCommander.newBuilder() - .addObject(options) - .build(); + .addObject(options) + .build(); commander.parse(args); // Print usage and exit if (options.help) { @@ -259,10 +258,10 @@ public static LoadOptions parseOptions(String[] args) { // Check options // Check option "-f" E.checkArgument(!StringUtils.isEmpty(options.file), - "The mapping file must be specified"); + "The mapping file must be specified"); E.checkArgument(options.file.endsWith(Constants.JSON_SUFFIX), - "The mapping file name must be end with %s", - Constants.JSON_SUFFIX); + "The mapping file name must be end with %s", + Constants.JSON_SUFFIX); File mappingFile = new File(options.file); if (!mappingFile.canRead()) { LOG.error("The mapping file must be readable: '{}'", mappingFile); @@ -271,7 +270,7 @@ public static LoadOptions parseOptions(String[] args) { // Check option "-g" E.checkArgument(!StringUtils.isEmpty(options.graph), - "The graph must be specified"); + "The graph must be specified"); // Check option "-h" if (!options.host.startsWith(Constants.HTTP_PREFIX)) { if (options.protocol.equals(HTTP_SCHEMA)) { @@ -282,7 +281,7 @@ public static LoadOptions parseOptions(String[] args) { } // Check option --incremental-mode and --failure-mode E.checkArgument(!(options.incrementalMode && options.failureMode), - "The option --incremental-mode and --failure-mode " + + "The option --incremental-mode and --failure-mode " + "can't be true at same time"); if (options.failureMode) { LOG.info("The failure-mode will scan the entire error file"); @@ -307,12 +306,12 @@ public static class UrlValidator implements IParameterValidator { @Override public void validate(String name, String value) { String regex = "^((http)(s?)://)?" + - "(([0-9]{1,3}\\.){3}[0-9]{1,3}" + // IP URL - "|" + // Or domain name - "([0-9a-z_!~*'()-]+\\.)*[0-9a-z_!~*'()-]+)$"; + "(([0-9]{1,3}\\.){3}[0-9]{1,3}" + // IP URL + "|" + // Or domain name + "([0-9a-z_!~*'()-]+\\.)*[0-9a-z_!~*'()-]+)$"; if (!value.matches(regex)) { throw new ParameterException(String.format( - "Invalid url value of args '%s': '%s'", name, value)); + "Invalid url value of args '%s': '%s'", name, value)); } } } @@ -327,8 +326,8 @@ public static class ProtocolValidator implements IParameterValidator { public void validate(String name, String value) { if (!SSL_PROTOCOL.contains(value.toLowerCase())) { throw new ParameterException(String.format( - "Invalid --protocol '%s', valid value is %s", - value, SSL_PROTOCOL)); + "Invalid --protocol '%s', valid value is %s", + value, SSL_PROTOCOL)); } } } @@ -353,8 +352,8 @@ public void validate(String name, String value) { File file = new File(value); if (!file.exists() || !file.isFile()) { throw new ParameterException(String.format( - "Ensure the file exists and is indeed a file " + - "instead of a directory: '%s'", value)); + "Ensure the file exists and is indeed a file " + + "instead of a directory: '%s'", value)); } } } From 78efb781dab742565a17910f0ae8870e7a1806f2 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Wed, 9 Nov 2022 10:37:05 +0800 Subject: [PATCH 21/25] adjust code style LoadOptions --- .../hugegraph/loader/executor/LoadOptions.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java index 7ba19e9d8..3d18865a7 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java @@ -76,17 +76,17 @@ public class LoadOptions implements Serializable { @Parameter(names = {"--protocol"}, arity = 1, validateWith = {ProtocolValidator.class}, description = "The protocol of HugeGraphServer, " + - "allowed values are: http or https") + "allowed values are: http or https") public String protocol = "http"; @Parameter(names = {"--trust-store-file"}, arity = 1, description = "The path of client truststore file used " + - "when https protocol is enabled") + "when https protocol is enabled") public String trustStoreFile = null; @Parameter(names = {"--trust-store-password"}, arity = 1, description = "The password of client truststore file used " + - "when https protocol is enabled") + "when https protocol is enabled") public String trustStoreToken = null; @Parameter(names = {"--token"}, arity = 1, @@ -155,19 +155,19 @@ public class LoadOptions implements Serializable { @Parameter(names = {"--max-read-errors"}, arity = 1, validateWith = {PositiveValidator.class}, description = "The maximum number of lines that read error " + - "before exiting") + "before exiting") public int maxReadErrors = 1; @Parameter(names = {"--max-parse-errors"}, arity = 1, validateWith = {PositiveValidator.class}, description = "The maximum number of lines that parse error " + - "before exiting") + "before exiting") public int maxParseErrors = 1; @Parameter(names = {"--max-insert-errors"}, arity = 1, validateWith = {PositiveValidator.class}, description = "The maximum number of lines that insert error " + - "before exiting") + "before exiting") public int maxInsertErrors = 500; @Parameter(names = {"--timeout"}, arity = 1, @@ -188,7 +188,7 @@ public class LoadOptions implements Serializable { @Parameter(names = {"--max-read-lines"}, arity = 1, validateWith = {PositiveValidator.class}, description = "The maximum number of read lines, when reached " + - "this number, the load task will stop") + "this number, the load task will stop") public long maxReadLines = -1L; @Parameter(names = {"--dry-run"}, arity = 1, From 6f66634871bbecd03a34b0af6331a5647fbd8a02 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 9 Nov 2022 18:51:43 +0800 Subject: [PATCH 22/25] adopt the apache commons v1.0 --- .../direct/loader/HBaseDirectLoader.java | 4 ++- .../loader/direct/util/SinkToHBase.java | 25 +++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java index 162b82d75..a5f1ce2b7 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java @@ -31,7 +31,8 @@ import com.baidu.hugegraph.structure.GraphElement; import com.baidu.hugegraph.structure.graph.Edge; import com.baidu.hugegraph.structure.graph.Vertex; -import com.baidu.hugegraph.util.Log; + +import org.apache.hugegraph.util.Log; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsShell; @@ -48,6 +49,7 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.slf4j.Logger; + import scala.Tuple2; import java.io.IOException; diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java index 6e04aa85d..b0f772178 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java @@ -19,32 +19,35 @@ package com.baidu.hugegraph.loader.direct.util; -import com.baidu.hugegraph.loader.executor.LoadOptions; -import com.baidu.hugegraph.util.Log; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.HRegionLocator; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hugegraph.util.Log; import org.apache.spark.Partitioner; import org.slf4j.Logger; + +import com.baidu.hugegraph.loader.executor.LoadOptions; + import scala.Tuple2; -import org.apache.hadoop.hbase.client.HRegionLocator; -import java.util.ArrayList; -import java.util.Arrays; -import java.io.IOException; -import java.util.HashMap; -import java.io.Serializable; -import java.util.List; -import java.util.Map; -import java.util.Optional; public class SinkToHBase implements Serializable { From dd7e290df2d4901fa253d3ab461de65244f13136 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Wed, 9 Nov 2022 22:41:33 +0800 Subject: [PATCH 23/25] adjust code style LoadOptions,SinkToHBase,HBaseDirectLoader... --- .../loader/direct/loader/HBaseDirectLoader.java | 1 - .../baidu/hugegraph/loader/direct/util/SinkToHBase.java | 9 +++------ .../com/baidu/hugegraph/loader/executor/LoadOptions.java | 2 +- .../com/baidu/hugegraph/loader/mapping/LoadMapping.java | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java index a5f1ce2b7..b2c10af92 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java @@ -265,5 +265,4 @@ private Tuple2 vertexSerialize (HBaseSerialize values); return new Tuple2<>(rowKey,keyValue); } - } diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java index b0f772178..99ea1d781 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/util/SinkToHBase.java @@ -91,7 +91,7 @@ private Optional getConnection() { new IntPartitioner(numPartitions, tableName),descriptor); } - public void loadHfiles (String path, String tableName) throws Exception { + public void loadHfiles(String path, String tableName) throws Exception { Connection conn = getConnection().get(); Table table = conn.getTable(TableName.valueOf(tableName)); Configuration conf = conn.getConfiguration(); @@ -136,14 +136,11 @@ public int numPartitions() { @Override public int getPartition(Object key) { if (key instanceof ImmutableBytesWritable) { - try { ImmutableBytesWritable immutableBytesWritableKey = (ImmutableBytesWritable) key; - if (rangeMap == null || rangeMap.isEmpty()) { rangeMap = getRangeMap(this.tableName); } - String keyString = Bytes.toString(immutableBytesWritableKey.get()); for (List range : rangeMap.keySet()) { if (keyString.compareToIgnoreCase(range.get(0)) >= 0 && @@ -155,8 +152,8 @@ public int getPartition(Object key) { LOG.error("Didn't find proper key in rangeMap, so returning 0 ..."); return 0; } catch (Exception e) { - LOG.error("When trying to get partitionID, encountered exception: " + e + - "\t key = " + key); + LOG.error("When trying to get partitionID, " + + "encountered exception: {} \t key = {}", e, key); return 0; } } else { diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java index 6843d8cf7..118b97373 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/executor/LoadOptions.java @@ -293,7 +293,7 @@ public static LoadOptions parseOptions(String[] args) { } public void copyBackendStoreInfo (BackendStoreInfo backendStoreInfo) { - E.checkArgument(null != backendStoreInfo,"the backendStoreInfo is null "); + E.checkArgument(null != backendStoreInfo, "The backendStoreInfo can't be null"); this.edgeTablename = backendStoreInfo.getEdgeTablename(); this.vertexTablename = backendStoreInfo.getVertexTablename(); this.hbaseZKParent = backendStoreInfo.getHbaseZKParent(); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java index 9a23905c6..c3afd7367 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/mapping/LoadMapping.java @@ -89,7 +89,7 @@ public LoadMapping(@JsonProperty("structs") List structs) { @JsonCreator public LoadMapping(@JsonProperty("structs") List structs, - @JsonProperty("backendStoreInfo") BackendStoreInfo backendStoreInfo) { + @JsonProperty("backendStoreInfo") BackendStoreInfo backendStoreInfo) { this.version = Constants.V2_STRUCT_VERSION; this.structs = structs; this.backendStoreInfo = backendStoreInfo; From 62a2e662a09a152d350d494d36502044267ca94c Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Wed, 9 Nov 2022 22:42:34 +0800 Subject: [PATCH 24/25] adjust code style HugrGraphSparkLoader,MappingUtil --- .../loader/spark/HugeGraphSparkLoader.java | 49 +++++++++---------- .../hugegraph/loader/util/MappingUtil.java | 2 +- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java index 160aa7777..acb188aee 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/spark/HugeGraphSparkLoader.java @@ -95,30 +95,27 @@ public void load() { LoadMapping mapping = LoadMapping.of(this.loadOptions.file); List structs = mapping.structs(); boolean sinkType = this.loadOptions.sinkType; - if(!sinkType){ + if(!sinkType) { this.loadOptions.copyBackendStoreInfo(mapping.getBackendStoreInfo()); } SparkConf conf = new SparkConf() .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")// kryo序列化 .set("spark.kryo.registrationRequired", "true"); try { - conf.registerKryoClasses( - new Class[] - { - org.apache.hadoop.hbase.io.ImmutableBytesWritable.class, - org.apache.hadoop.hbase.KeyValue.class, - org.apache.spark.sql.types.StructType.class, - StructField[].class, - StructField.class, - org.apache.spark.sql.types.LongType$.class, - org.apache.spark.sql.types.Metadata.class, - org.apache.spark.sql.types.StringType$.class, - Class.forName( - "org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage"), - Class.forName("scala.reflect.ClassTag$$anon$1"), - Class.forName("scala.collection.immutable.Set$EmptySet$"), - Class.forName("org.apache.spark.sql.types.DoubleType$") - }); + conf.registerKryoClasses(new Class[] { + org.apache.hadoop.hbase.io.ImmutableBytesWritable.class, + org.apache.hadoop.hbase.KeyValue.class, + org.apache.spark.sql.types.StructType.class, + StructField[].class, + StructField.class, + org.apache.spark.sql.types.LongType$.class, + org.apache.spark.sql.types.Metadata.class, + org.apache.spark.sql.types.StringType$.class, + Class.forName("org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage"), + Class.forName("scala.reflect.ClassTag$$anon$1"), + Class.forName("scala.collection.immutable.Set$EmptySet$"), + Class.forName("org.apache.spark.sql.types.DoubleType$") + }); } catch (ClassNotFoundException e) { LOG.error("spark kryo serialized registration failed"); } @@ -130,11 +127,11 @@ public void load() { LongAccumulator totalInsertSuccess = sc.longAccumulator("totalInsertSuccess"); for (InputStruct struct : structs) { LOG.info("\n Initializes the accumulator corresponding to the {} ", - struct.input().asFileSource().path()); + struct.input().asFileSource().path()); LoadDistributeMetrics loadDistributeMetrics = new LoadDistributeMetrics(struct); loadDistributeMetrics.init(sc); - LOG.info("\n Start to load data, data info is: \t {} ", - struct.input().asFileSource().path()); + LOG.info("\n Start to load data, data info is: \t {} ", + struct.input().asFileSource().path()); Dataset ds = read(session, struct); if (sinkType) { LOG.info("\n Start to load data using spark apis \n"); @@ -155,14 +152,14 @@ public void load() { } collectLoadMetrics(loadDistributeMetrics,totalInsertSuccess); - LOG.info(" \n Finished load {} data ", - struct.input().asFileSource().path()); + LOG.info("\n Finished load {} data ", + struct.input().asFileSource().path()); } Long totalInsertSuccessCnt = totalInsertSuccess.value(); LOG.info("\n ------------The data load task is complete-------------------\n" + - "\n insertSuccesscnt:\t {}" + - "\n ---------------------------------------------\n" - , totalInsertSuccessCnt); + "\n insertSuccesscnt:\t {}" + + "\n ---------------------------------------------\n" + , totalInsertSuccessCnt); sc.stop(); session.close(); diff --git a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java index ca88f40c9..a9667806f 100644 --- a/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java +++ b/hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/util/MappingUtil.java @@ -129,7 +129,7 @@ private static LoadMapping parseV1(String json) { inputStruct.id(String.valueOf(++id)); inputStructs.add(inputStruct); } - return new LoadMapping(inputStructs,graphStruct.getBackendStoreInfo()); + return new LoadMapping(inputStructs, graphStruct.getBackendStoreInfo()); } private static ElementMapping convertV1ToV2(ElementStructV1 origin) { From e3c8a9046f3a7db83f062c370623f2bce5f36634 Mon Sep 17 00:00:00 2001 From: "alan.zhao" <956322745@qq.com> Date: Thu, 10 Nov 2022 13:16:45 +0800 Subject: [PATCH 25/25] MappingConverterTest add backendstoreinfo,fix hugegraph-loader-ci check faield --- .../loader/test/unit/MappingConverterTest.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/hugegraph-loader/src/test/java/com/baidu/hugegraph/loader/test/unit/MappingConverterTest.java b/hugegraph-loader/src/test/java/com/baidu/hugegraph/loader/test/unit/MappingConverterTest.java index c937daed0..c2a14b018 100644 --- a/hugegraph-loader/src/test/java/com/baidu/hugegraph/loader/test/unit/MappingConverterTest.java +++ b/hugegraph-loader/src/test/java/com/baidu/hugegraph/loader/test/unit/MappingConverterTest.java @@ -73,7 +73,15 @@ public void testConvertV1ToV2() throws IOException { " \"Rating\": \"rate\"" + " }" + " }" + - " ]" + + " ]," + + " \"backendStoreInfo\":" + + " {" + + " \"edge_tablename\": \"hugegraph:g_oe\"," + + " \"vertex_tablename\": \"hugegraph:g_v\"," + + " \"hbase_zookeeper_quorum\": \"127.0.0.1\"," + + " \"hbase_zookeeper_property_clientPort\": \"2181\"," + + " \"zookeeper_znode_parent\": \"/hbase\"" + + " }" + "}"; String input = "struct.json"; File inputFile = new File(input); @@ -117,7 +125,13 @@ public void testConvertV1ToV2() throws IOException { "\"field_mapping\":{\"UserID\":\"id\",\"MovieID\":\"id\"," + "\"Rating\":\"rate\"},\"value_mapping\":{},\"selected\":[]," + "\"ignored\":[\"Timestamp\"],\"null_values\":[\"\"]," + - "\"update_strategies\":{},\"batch_size\":500}]}]}"; + "\"update_strategies\":{},\"batch_size\":500}]}]," + + "\"backendStoreInfo\":{" + + "\"edge_tablename\":\"hugegraph:g_oe\"," + + "\"vertex_tablename\":\"hugegraph:g_v\"," + + "\"hbase_zookeeper_quorum\":\"127.0.0.1\"," + + "\"hbase_zookeeper_property_clientPort\":\"2181\"," + + "\"zookeeper_znode_parent\":\"/hbase\"}}"; Assert.assertEquals(expectV2Json, actualV2Json); FileUtils.forceDelete(inputFile);