Wednesday, August 28, 2013

Hive Connection with MongoDB

Do the processing by hive and store the result into MongoDB.

ADD JAR /home/hadoop/Documents/Hive-mongo-master/release/hive-mongo-0.0.2.jar;
ADD JAR /home/hadoop/Documents/Hive-mongo-master/release/hive-mongo-0.0.2-jar-with-dependencies.jar;
ADD JAR /usr/lib/hive-0.10.0/lib/hive-metastore-0.10.0.jar;
ADD JAR /home/hadoop/Documents/guava-r06.jar;
ADD JAR /home/hadoop/Documents/mongo-java-driver-2.6.3.jar;
create table if not exists mongo_test(id int, name String, age int) ROW FORMAT DELIMITED 
FIELDS TERMINATED BY ',';

 load data local inpath '/home/hadoop/Documents/mongodata.txt' into table mongo_test;

 create external table mongotable(id int, name string, age int)
     stored by "org.yong3.hive.mongo.MongoStorageHandler"
     with serdeproperties ( "mongo.column.mapping" = "_id,name,age" )
     tblproperties ( "mongo.host" = "localhost", "mongo.port" = "27017",
    "mongo.db" = "db", "mongo.collection" = "ravi" );

 insert overwrite table mongotable select id, name, age from mongo_test;

1 comment:

  1. Hi Ravi,

    I did the same thing which has mentioned above, but i am getting the following error whenever i try to perform 'insert overwrite' on mongotable. Could you please tell the solution for it. Thanks in Advance.

    java.lang.AbstractMethodError: org.yong3.hive.mongo.MongoStorageHandler.configureJobConf(Lorg/apache/hadoop/hive/ql/plan/TableDesc;Lorg/apache/hadoop/mapred/JobConf;)V
    at org.apache.hadoop.hive.ql.plan.PlanUtils.configureJobConf(PlanUtils.java:800)
    at org.apache.hadoop.hive.ql.plan.MapWork.configureJobConf(MapWork.java:479)
    at org.apache.hadoop.hive.ql.plan.MapredWork.configureJobConf(MapredWork.java:70)
    at org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(ExecDriver.java:379)
    at org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(MapRedTask.java:144)
    at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:151)
    at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:65)
    at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1414)
    at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1192)
    at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1020)
    at org.apache.hadoop.hive.ql.Driver.run(Driver.java:888)
    at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:259)
    at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:216)
    at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:413)
    at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:781)
    at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:675)
    at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:614)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:606)
    at org.apache.hadoop.util.RunJar.main(RunJar.java:212)
    FAILED: Execution Error, return code -101 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask

    ReplyDelete