Configure HDFS NFS Gateway
[root@cdh60 ~]# showmount -e cdh60
Export list for cdh60:
/ *
[root@cdh60 ~]# rpcinfo cdh60
program version netid address service owner
100000 4 tcp6 ::.0.111 portmapper superuser
100000 3 tcp6 ::.0.111 portmapper superuser
100000 4 udp6 ::.0.111 portmapper superuser
100000 3 udp6 ::.0.111 portmapper superuser
100000 4 tcp 0.0.0.0.0.111 portmapper superuser
100000 3 tcp 0.0.0.0.0.111 portmapper superuser
100000 2 tcp 0.0.0.0.0.111 portmapper superuser
100000 4 udp 0.0.0.0.0.111 portmapper superuser
100000 3 udp 0.0.0.0.0.111 portmapper superuser
100000 2 udp 0.0.0.0.0.111 portmapper superuser
100000 4 local /var/run/rpcbind.sock portmapper superuser
100000 3 local /var/run/rpcbind.sock portmapper superuser
100005 1 udp 0.0.0.0.16.146 mountd superuser
100005 2 udp 0.0.0.0.16.146 mountd superuser
100005 3 udp 0.0.0.0.16.146 mountd superuser
100005 1 tcp 0.0.0.0.16.146 mountd superuser
100005 2 tcp 0.0.0.0.16.146 mountd superuser
100005 3 tcp 0.0.0.0.16.146 mountd superuser
100003 3 tcp 0.0.0.0.8.1 nfs superuser
[root@cdh60 ~]# mkdir /hdfs_nfs_mount
root@cdh60 ~]# mount -t nfs -o vers=3,proto=tcp,nolock cdh60:/ /hdfs_nfs_mount
[root@cdh60 ~]# df -h /hdfs_nfs_mount
Filesystem Size Used Avail Use% Mounted on
cdh60:/ 69G 6.5G 63G 10% /hdfs_nfs_mount
Linux Permission Applies:
[root@cdh60 dsuser]# ls -ld /hdfs_nfs_mount/data/incoming/
drwxr-xr-x 3 dsuser 2584148964 96 Aug 19 12:17 /hdfs_nfs_mount/data/incoming/
[root@cdh60 dsuser]# cp employees.csv /hdfs_nfs_mount/data/incoming/
cp: cannot create regular file ‘/hdfs_nfs_mount/data/incoming/employees.csv’: Permission denied
Login as dsuser (owner of incoming folder)
Donghuas-MacBook-Air:pandas donghua$ ssh dsuser@cdh60
dsuser@cdh60's password:
Last login: Sun Aug 19 09:24:50 2018 from 192.168.1.1
[dsuser@cdh60 ~]$ klist
Ticket cache: FILE:/tmp/krb5cc_1001
Default principal: dsuser@DBAGLOBE.COM
Valid starting Expires Service principal
08/19/2018 09:24:54 08/20/2018 09:24:54 krbtgt/DBAGLOBE.COM@DBAGLOBE.COM
renew until 08/26/2018 09:24:54
[dsuser@cdh60 ~]$ cp employees.csv /hdfs_nfs_mount/data/incoming/
[dsuser@cdh60 ~]$ ls -l /hdfs_nfs_mount/data/incoming/employees.csv
-rw-r--r-- 1 dsuser 2584148964 59175 Aug 19 12:17 /hdfs_nfs_mount/data/incoming/employees.csv
[dsuser@cdh60 ~]$ hdfs dfs -ls /data/incoming/
Found 1 items
-rw-r--r-- 1 dsuser supergroup 59175 2018-08-19 12:17 /data/incoming/employees.csv
[dsuser@cdh60 ~]$ hdfs dfs -cat /data/incoming/employees.csv|head -n 3
First Name,Gender,Start Date,Last Login Time,Salary,Bonus %,Senior Management,Team
Douglas,Male,8/6/1993,12:42 PM,97308,6.945,true,Marketing
Thomas,Male,3/31/1996,6:53 AM,61933,4.17,true,
[dsuser@cdh60 ~]$ cat /hdfs_nfs_mount/data/incoming/employees.csv |head -n3
First Name,Gender,Start Date,Last Login Time,Salary,Bonus %,Senior Management,Team
Douglas,Male,8/6/1993,12:42 PM,97308,6.945,true,Marketing
Thomas,Male,3/31/1996,6:53 AM,61933,4.17,true,
Login as root (act as normal nfs client user)
[root@cdh60 dsuser]# hdfs dfs -cat /data/incoming/employees.csv|head -n 3
18/08/19 12:26:56 WARN ipc.Client: Exception encountered while connecting to the server : org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]
cat: Failed on local exception: java.io.IOException: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]; Host Details : local host is: "cdh60.dbaglobe.com/192.168.56.110"; destination host is: "cdh60.dbaglobe.com":8020;
[root@cdh60 dsuser]# cat /hdfs_nfs_mount/data/incoming/employees.csv |head -n3
First Name,Gender,Start Date,Last Login Time,Salary,Bonus %,Senior Management,Team
Douglas,Male,8/6/1993,12:42 PM,97308,6.945,true,Marketing
Thomas,Male,3/31/1996,6:53 AM,61933,4.17,true,
Showing posts with label Data Lake. Show all posts
Showing posts with label Data Lake. Show all posts
Sunday, August 19, 2018
Wednesday, March 7, 2018
Construct SCD Type 2 in Hive examples
create table products(prod_id int, name string, price double, last_modified timestamp) stored as parquet;
create table orders(order_id int, prod_id int, quantity int, order_date timestamp) stored as parquet;
insert into products values(1,'Soap',1.0,'2017-01-01');
insert into products values(1,'Soap',1.1,'2017-04-01');
insert into products values(1,'Soap',1.2,'2017-07-01');
insert into products values(1,'Soap',1.3,'2017-10-01');
insert into products values(2,'Soda',0.6,'2017-01-01');
insert into products values(2,'Soda',0.8,'2017-06-01');
insert into products values(3,'Beer',4.2,'2017-07-01');
insert into orders values(1,1,1,'2017-02-10');
insert into orders values(2,2,10,'2017-07-10');
insert into orders values(3,3,5,'2017-09-10');
insert into orders values(4,1,2,'2017-12-01');
create view prod_scd2 as
select prod_id,price,last_modified as valid_from,
lead(last_modified) over (partition by prod_id order by last_modified) as valid_end
from products;
0: jdbc:hive2://cdh-vm.dbaglobe.com:10000> select * from prod_scd2 p;
+------------+----------+------------------------+------------------------+--+
| p.prod_id | p.price | p.valid_from | p.valid_end |
+------------+----------+------------------------+------------------------+--+
| 1 | 1.0 | 2017-01-01 00:00:00.0 | 2017-04-01 00:00:00.0 |
| 1 | 1.1 | 2017-04-01 00:00:00.0 | 2017-07-01 00:00:00.0 |
| 1 | 1.2 | 2017-07-01 00:00:00.0 | 2017-10-01 00:00:00.0 |
| 1 | 1.3 | 2017-10-01 00:00:00.0 | NULL |
| 2 | 0.6 | 2017-01-01 00:00:00.0 | 2017-06-01 00:00:00.0 |
| 2 | 0.8 | 2017-06-01 00:00:00.0 | NULL |
| 3 | 4.2 | 2017-07-01 00:00:00.0 | NULL |
+------------+----------+------------------------+------------------------+--+
7 rows selected (20.253 seconds)
0: jdbc:hive2://cdh-vm.dbaglobe.com:10000> select * from orders o;
+-------------+------------+-------------+------------------------+--+
| o.order_id | o.prod_id | o.quantity | o.order_date |
+-------------+------------+-------------+------------------------+--+
| 1 | 1 | 1 | 2017-02-10 00:00:00.0 |
| 2 | 2 | 10 | 2017-07-10 00:00:00.0 |
| 3 | 3 | 5 | 2017-09-10 00:00:00.0 |
| 4 | 1 | 2 | 2017-12-01 00:00:00.0 |
+-------------+------------+-------------+------------------------+--+
4 rows selected (0.152 seconds)
select o.order_id,o.order_date,p.price,o.quantity*p.price as total_cost
from orders o join prod_scd2 p
on (o.prod_id=p.prod_id)
where o.order_date between p.valid_from and if(p.valid_end is not null ,p.valid_end,cast('9999-12-31' as timestamp))
order by order_id;
0: jdbc:hive2://cdh-vm.dbaglobe.com:10000> select o.order_id,o.order_date,p.price,o.quantity*p.price as total_cost
. . . . . . . . . . . . . . . . . . . . .> from orders o join prod_scd2 p
. . . . . . . . . . . . . . . . . . . . .> on (o.prod_id=p.prod_id)
. . . . . . . . . . . . . . . . . . . . .> where o.order_date between p.valid_from and if(p.valid_end is not null ,p.valid_end,cast('9999-12-31' as timestamp))
. . . . . . . . . . . . . . . . . . . . .> order by order_id;
+-------------+------------------------+----------+-------------+--+
| o.order_id | o.order_date | p.price | total_cost |
+-------------+------------------------+----------+-------------+--+
| 1 | 2017-02-10 00:00:00.0 | 1.0 | 1.0 |
| 2 | 2017-07-10 00:00:00.0 | 0.8 | 8.0 |
| 3 | 2017-09-10 00:00:00.0 | 4.2 | 21.0 |
| 4 | 2017-12-01 00:00:00.0 | 1.3 | 2.6 |
+-------------+------------------------+----------+-------------+--+
4 rows selected (48.862 seconds)
create table orders(order_id int, prod_id int, quantity int, order_date timestamp) stored as parquet;
insert into products values(1,'Soap',1.0,'2017-01-01');
insert into products values(1,'Soap',1.1,'2017-04-01');
insert into products values(1,'Soap',1.2,'2017-07-01');
insert into products values(1,'Soap',1.3,'2017-10-01');
insert into products values(2,'Soda',0.6,'2017-01-01');
insert into products values(2,'Soda',0.8,'2017-06-01');
insert into products values(3,'Beer',4.2,'2017-07-01');
insert into orders values(1,1,1,'2017-02-10');
insert into orders values(2,2,10,'2017-07-10');
insert into orders values(3,3,5,'2017-09-10');
insert into orders values(4,1,2,'2017-12-01');
create view prod_scd2 as
select prod_id,price,last_modified as valid_from,
lead(last_modified) over (partition by prod_id order by last_modified) as valid_end
from products;
0: jdbc:hive2://cdh-vm.dbaglobe.com:10000> select * from prod_scd2 p;
+------------+----------+------------------------+------------------------+--+
| p.prod_id | p.price | p.valid_from | p.valid_end |
+------------+----------+------------------------+------------------------+--+
| 1 | 1.0 | 2017-01-01 00:00:00.0 | 2017-04-01 00:00:00.0 |
| 1 | 1.1 | 2017-04-01 00:00:00.0 | 2017-07-01 00:00:00.0 |
| 1 | 1.2 | 2017-07-01 00:00:00.0 | 2017-10-01 00:00:00.0 |
| 1 | 1.3 | 2017-10-01 00:00:00.0 | NULL |
| 2 | 0.6 | 2017-01-01 00:00:00.0 | 2017-06-01 00:00:00.0 |
| 2 | 0.8 | 2017-06-01 00:00:00.0 | NULL |
| 3 | 4.2 | 2017-07-01 00:00:00.0 | NULL |
+------------+----------+------------------------+------------------------+--+
7 rows selected (20.253 seconds)
0: jdbc:hive2://cdh-vm.dbaglobe.com:10000> select * from orders o;
+-------------+------------+-------------+------------------------+--+
| o.order_id | o.prod_id | o.quantity | o.order_date |
+-------------+------------+-------------+------------------------+--+
| 1 | 1 | 1 | 2017-02-10 00:00:00.0 |
| 2 | 2 | 10 | 2017-07-10 00:00:00.0 |
| 3 | 3 | 5 | 2017-09-10 00:00:00.0 |
| 4 | 1 | 2 | 2017-12-01 00:00:00.0 |
+-------------+------------+-------------+------------------------+--+
4 rows selected (0.152 seconds)
select o.order_id,o.order_date,p.price,o.quantity*p.price as total_cost
from orders o join prod_scd2 p
on (o.prod_id=p.prod_id)
where o.order_date between p.valid_from and if(p.valid_end is not null ,p.valid_end,cast('9999-12-31' as timestamp))
order by order_id;
0: jdbc:hive2://cdh-vm.dbaglobe.com:10000> select o.order_id,o.order_date,p.price,o.quantity*p.price as total_cost
. . . . . . . . . . . . . . . . . . . . .> from orders o join prod_scd2 p
. . . . . . . . . . . . . . . . . . . . .> on (o.prod_id=p.prod_id)
. . . . . . . . . . . . . . . . . . . . .> where o.order_date between p.valid_from and if(p.valid_end is not null ,p.valid_end,cast('9999-12-31' as timestamp))
. . . . . . . . . . . . . . . . . . . . .> order by order_id;
+-------------+------------------------+----------+-------------+--+
| o.order_id | o.order_date | p.price | total_cost |
+-------------+------------------------+----------+-------------+--+
| 1 | 2017-02-10 00:00:00.0 | 1.0 | 1.0 |
| 2 | 2017-07-10 00:00:00.0 | 0.8 | 8.0 |
| 3 | 2017-09-10 00:00:00.0 | 4.2 | 21.0 |
| 4 | 2017-12-01 00:00:00.0 | 1.3 | 2.6 |
+-------------+------------------------+----------+-------------+--+
4 rows selected (48.862 seconds)
Thursday, December 29, 2016
Access HBase from BDA Lite using Oracle BigData SQL
SQL> select * from hbase_datademo where rowkey=1000001451580500;
select * from hbase_datademo where rowkey=1000001451580500
*
ERROR at line 1:
ORA-29913: error in executing ODCIEXTTABLEOPEN callout
ORA-29400: data cartridge error
KUP-11504: error from external driver: java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
[oracle@bigdatalite bigdatasql_config]$ less /tmp/bigdata-log4j.log
16/12/29 11:25:41 INFO xcat.XCat: cluster property com.oracle.bigdata.conf.default.serdepropcacheenabled is true
16/12/29 11:25:41 DEBUG xcat.XCat: access param com.oracle.bigdata.serdepropcache is true
16/12/29 11:25:41 DEBUG hive.XCatHive: XCatHive.open: connecting to the hive metastore...
16/12/29 11:25:41 DEBUG hive.XCatHive: XCatHive.open: connect succeeded
16/12/29 11:25:41 DEBUG hive.XCatHive: XCatHive.open: getTable succeeded
16/12/29 11:25:41 ERROR sql.JXADDescribe: JXADDescribe.getInfo
java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
at org.apache.hadoop.hive.hbase.HBaseSerDe.parseColumnsMapping(HBaseSerDe.java:184)
at org.apache.hadoop.hive.hbase.HBaseSerDeParameters.(HBaseSerDeParameters.java:73)
at org.apache.hadoop.hive.hbase.HBaseSerDe.initialize(HBaseSerDe.java:117)
at org.apache.hadoop.hive.serde2.AbstractSerDe.initialize(AbstractSerDe.java:53)
at org.apache.hadoop.hive.serde2.SerDeUtils.initializeSerDe(SerDeUtils.java:520)
at org.apache.hadoop.hive.metastore.MetaStoreUtils.getDeserializer(MetaStoreUtils.java:392)
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializerFromMetaStore(Table.java:273)
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializer(Table.java:255)
at org.apache.hadoop.hive.ql.metadata.Table.getCols(Table.java:602)
at oracle.hadoop.sql.xcat.hive.XCatHiveUtil.extractSchema(XCatHiveUtil.java:205)
at oracle.hadoop.sql.xcat.hive.XCatHiveUtil.valueOf(XCatHiveUtil.java:276)
at oracle.hadoop.sql.xcat.hive.XCatHive.open(XCatHive.java:209)
at oracle.hadoop.sql.JXADDescribe._compile(JXADDescribe.java:491)
at oracle.hadoop.sql.JXADDescribe.getXaddoc(JXADDescribe.java:621)
at oracle.hadoop.sql.JXADDescribe._getInfo(JXADDescribe.java:429)
at oracle.hadoop.sql.JXADDescribe.getInfo(JXADDescribe.java:380)
16/12/29 11:25:41 DEBUG metrics.TaskMetrics: jxad_task: DESCRIBE__GET_XADDOC_REQTYPE, time: 64, bytes: 0
16/12/29 11:25:41 TRACE sql.JXADProvider: Entering kubsagtiJNIConvCB...
16/12/29 11:25:41 TRACE sql.JXADProvider: Leaving kubsagtiJNIConvCB...retSize=0
16/12/29 11:25:41 TRACE sql.JXADProvider: java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
16/12/29 11:25:41 ERROR sql.JXADProvider: KUP-11504: error from external driver: java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
16/12/29 11:25:41 TRACE sql.JXADProvider: Leaving kubsjniDescribeGetInfo...status=6
16/12/29 11:25:41 TRACE sql.JXADProvider: Leaving kubsagtiDescribeGetInfo...status=33
16/12/29 11:25:41 TRACE sql.JXADProvider: Leaving kubsagtDescribeGetInfo...status=33
16/12/29 11:25:41 DEBUG sql.JXADProvider: using agent handle=79
16/12/29 11:25:41 DEBUG sql.JXADProvider: using agent context=0xedfe98
16/12/29 11:25:41 DEBUG sql.JXADProvider: using agent jvm pointer=0x7fe020bf1b48
16/12/29 11:25:41 DEBUG sql.JXADProvider: using agent jni handle=0x10009f8
SQL> select * from hbase_datademo where rownum<10 p="">select * from hbase_datademo where rownum<10 p=""> *
ERROR at line 1:
ORA-29913: error in executing ODCIEXTTABLEOPEN callout
ORA-29400: data cartridge error
KUP-11504: error from external driver:
oracle.hadoop.sql.xcat.common.XCatException : 1001 : Error getting splits
document. Cause : java.io.IOException:
java.lang.reflect.InvocationTargetException
16/12/29 13:40:15 DEBUG hadoop.XCatInfm: listStatus(final) file =hdfs://bigdatalite.localdomain:8020/user/hive/warehouse/hbase_datademo
16/12/29 13:40:15 ERROR sql.JXADDescribe: JXADDescribe.getInfo
oracle.hadoop.sql.xcat.common.XCatException : 1001 : Error getting splits document. Cause : java.io.IOException: java.lang.reflect.InvocationTargetException
at oracle.hadoop.sql.xcat.XCatInfo.getSplitsDoc(XCatInfo.java:395)
at oracle.hadoop.sql.JXADDescribe.getSplitsDoc(JXADDescribe.java:545)
at oracle.hadoop.sql.JXADDescribe._getInfo(JXADDescribe.java:417)
at oracle.hadoop.sql.JXADDescribe.getInfo(JXADDescribe.java:380)
Caused by: java.io.IOException: java.lang.reflect.InvocationTargetException
at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:240)
at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:218)
at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:119)
at org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat.getSplitsInternal(HiveHBaseTableInputFormat.java:463)
at org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat.getSplits(HiveHBaseTableInputFormat.java:450)
at oracle.hadoop.sql.xcat.hadoop.mapred.XCatMapredInfm.computeBaseSplits(XCatMapredInfm.java:169)
at oracle.hadoop.sql.xcat.hadoop.XCatGetSplits.computeSplits(XCatGetSplits.java:563)
at oracle.hadoop.sql.xcat.XCatInfo.getSplits(XCatInfo.java:354)
at oracle.hadoop.sql.xcat.XCatInfo.getSplitsDoc(XCatInfo.java:384)
... 3 more
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)
... 11 more
Caused by: java.lang.NoClassDefFoundError: org/apache/htrace/Trace
at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:217)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:419)
at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:65)
at org.apache.hadoop.hbase.client.ZooKeeperRegistry.getClusterId(ZooKeeperRegistry.java:105)
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.retrieveClusterId(ConnectionManager.java:919)
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.(ConnectionManager.java:657)
... 16 more
16/12/29 13:40:15 DEBUG metrics.TaskMetrics: jxad_task: DESCRIBE__GET_SPLITS_DOC_REQTYPE, time: 59, bytes: 0
16/12/29 13:40:15 TRACE sql.JXADProvider: Entering kubsagtiJNIConvCB...
How to Fix: Adding Hbase and Hbase Lib to java class path for hadoop inside bigdata.properties file
[oracle@bigdatalite tmp]$ diff /u01/bigdatasql_config/bigdata.properties /u01/bigdatasql_config/bigdata.properties.orig
3c3
< java.classpath.hadoop=/usr/lib/hadoop/client/*:/usr/lib/hadoop-mapreduce/*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hbase/*:/usr/lib/hbase/lib/*
---
> java.classpath.hadoop=/usr/lib/hadoop/client/*:/usr/lib/hadoop-mapreduce/*:/usr/lib/hadoop-mapreduce/lib/*
--- restart BDA Lite
select * from hbase_datademo where rowkey=1000001451580500
*
ERROR at line 1:
ORA-29913: error in executing ODCIEXTTABLEOPEN callout
ORA-29400: data cartridge error
KUP-11504: error from external driver: java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
[oracle@bigdatalite bigdatasql_config]$ less /tmp/bigdata-log4j.log
16/12/29 11:25:41 INFO xcat.XCat: cluster property com.oracle.bigdata.conf.default.serdepropcacheenabled is true
16/12/29 11:25:41 DEBUG xcat.XCat: access param com.oracle.bigdata.serdepropcache is true
16/12/29 11:25:41 DEBUG hive.XCatHive: XCatHive.open: connecting to the hive metastore...
16/12/29 11:25:41 DEBUG hive.XCatHive: XCatHive.open: connect succeeded
16/12/29 11:25:41 DEBUG hive.XCatHive: XCatHive.open: getTable succeeded
16/12/29 11:25:41 ERROR sql.JXADDescribe: JXADDescribe.getInfo
java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
at org.apache.hadoop.hive.hbase.HBaseSerDe.parseColumnsMapping(HBaseSerDe.java:184)
at org.apache.hadoop.hive.hbase.HBaseSerDeParameters.
at org.apache.hadoop.hive.hbase.HBaseSerDe.initialize(HBaseSerDe.java:117)
at org.apache.hadoop.hive.serde2.AbstractSerDe.initialize(AbstractSerDe.java:53)
at org.apache.hadoop.hive.serde2.SerDeUtils.initializeSerDe(SerDeUtils.java:520)
at org.apache.hadoop.hive.metastore.MetaStoreUtils.getDeserializer(MetaStoreUtils.java:392)
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializerFromMetaStore(Table.java:273)
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializer(Table.java:255)
at org.apache.hadoop.hive.ql.metadata.Table.getCols(Table.java:602)
at oracle.hadoop.sql.xcat.hive.XCatHiveUtil.extractSchema(XCatHiveUtil.java:205)
at oracle.hadoop.sql.xcat.hive.XCatHiveUtil.valueOf(XCatHiveUtil.java:276)
at oracle.hadoop.sql.xcat.hive.XCatHive.open(XCatHive.java:209)
at oracle.hadoop.sql.JXADDescribe._compile(JXADDescribe.java:491)
at oracle.hadoop.sql.JXADDescribe.getXaddoc(JXADDescribe.java:621)
at oracle.hadoop.sql.JXADDescribe._getInfo(JXADDescribe.java:429)
at oracle.hadoop.sql.JXADDescribe.getInfo(JXADDescribe.java:380)
16/12/29 11:25:41 DEBUG metrics.TaskMetrics: jxad_task: DESCRIBE__GET_XADDOC_REQTYPE, time: 64, bytes: 0
16/12/29 11:25:41 TRACE sql.JXADProvider: Entering kubsagtiJNIConvCB...
16/12/29 11:25:41 TRACE sql.JXADProvider: Leaving kubsagtiJNIConvCB...retSize=0
16/12/29 11:25:41 TRACE sql.JXADProvider: java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
16/12/29 11:25:41 ERROR sql.JXADProvider: KUP-11504: error from external driver: java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
16/12/29 11:25:41 TRACE sql.JXADProvider: Leaving kubsjniDescribeGetInfo...status=6
16/12/29 11:25:41 TRACE sql.JXADProvider: Leaving kubsagtiDescribeGetInfo...status=33
16/12/29 11:25:41 TRACE sql.JXADProvider: Leaving kubsagtDescribeGetInfo...status=33
16/12/29 11:25:41 DEBUG sql.JXADProvider: using agent handle=79
16/12/29 11:25:41 DEBUG sql.JXADProvider: using agent context=0xedfe98
16/12/29 11:25:41 DEBUG sql.JXADProvider: using agent jvm pointer=0x7fe020bf1b48
16/12/29 11:25:41 DEBUG sql.JXADProvider: using agent jni handle=0x10009f8
SQL> select * from hbase_datademo where rownum<10 p="">select * from hbase_datademo where rownum<10 p=""> *
ERROR at line 1:
ORA-29913: error in executing ODCIEXTTABLEOPEN callout
ORA-29400: data cartridge error
KUP-11504: error from external driver:
oracle.hadoop.sql.xcat.common.XCatException : 1001 : Error getting splits
document. Cause : java.io.IOException:
java.lang.reflect.InvocationTargetException
16/12/29 13:40:15 DEBUG hadoop.XCatInfm: listStatus(final) file =hdfs://bigdatalite.localdomain:8020/user/hive/warehouse/hbase_datademo
16/12/29 13:40:15 ERROR sql.JXADDescribe: JXADDescribe.getInfo
oracle.hadoop.sql.xcat.common.XCatException : 1001 : Error getting splits document. Cause : java.io.IOException: java.lang.reflect.InvocationTargetException
at oracle.hadoop.sql.xcat.XCatInfo.getSplitsDoc(XCatInfo.java:395)
at oracle.hadoop.sql.JXADDescribe.getSplitsDoc(JXADDescribe.java:545)
at oracle.hadoop.sql.JXADDescribe._getInfo(JXADDescribe.java:417)
at oracle.hadoop.sql.JXADDescribe.getInfo(JXADDescribe.java:380)
Caused by: java.io.IOException: java.lang.reflect.InvocationTargetException
at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:240)
at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:218)
at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:119)
at org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat.getSplitsInternal(HiveHBaseTableInputFormat.java:463)
at org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat.getSplits(HiveHBaseTableInputFormat.java:450)
at oracle.hadoop.sql.xcat.hadoop.mapred.XCatMapredInfm.computeBaseSplits(XCatMapredInfm.java:169)
at oracle.hadoop.sql.xcat.hadoop.XCatGetSplits.computeSplits(XCatGetSplits.java:563)
at oracle.hadoop.sql.xcat.XCatInfo.getSplits(XCatInfo.java:354)
at oracle.hadoop.sql.xcat.XCatInfo.getSplitsDoc(XCatInfo.java:384)
... 3 more
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)
... 11 more
Caused by: java.lang.NoClassDefFoundError: org/apache/htrace/Trace
at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:217)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:419)
at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:65)
at org.apache.hadoop.hbase.client.ZooKeeperRegistry.getClusterId(ZooKeeperRegistry.java:105)
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.retrieveClusterId(ConnectionManager.java:919)
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.
... 16 more
16/12/29 13:40:15 DEBUG metrics.TaskMetrics: jxad_task: DESCRIBE__GET_SPLITS_DOC_REQTYPE, time: 59, bytes: 0
16/12/29 13:40:15 TRACE sql.JXADProvider: Entering kubsagtiJNIConvCB...
How to Fix: Adding Hbase and Hbase Lib to java class path for hadoop inside bigdata.properties file
[oracle@bigdatalite tmp]$ diff /u01/bigdatasql_config/bigdata.properties /u01/bigdatasql_config/bigdata.properties.orig
3c3
< java.classpath.hadoop=/usr/lib/hadoop/client/*:/usr/lib/hadoop-mapreduce/*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hbase/*:/usr/lib/hbase/lib/*
---
> java.classpath.hadoop=/usr/lib/hadoop/client/*:/usr/lib/hadoop-mapreduce/*:/usr/lib/hadoop-mapreduce/lib/*
--- restart BDA Lite
Wednesday, June 1, 2016
sqoop examples
[cloudera@quickstart ~]$ sqoop import-all-tables \
-m 1 \
--connect jdbc:mysql://quickstart:3306/retail_db \
--username=retail_dba \
--password=cloudera \
--compression-codec=snappy \
--as-parquetfile \
--warehouse-dir=/user/hive/warehouse \
--hive-import
=========================================================
[cloudera@quickstart ~]$ ps -ef|grep mysql
root 1950 1 0 06:28 ? 00:00:00 /bin/sh /usr/bin/mysqld_safe --datadir=/var/lib/mysql --socket=/var/lib/mysql/mysql.sock --pid-file=/var/run/mysqld/mysqld.pid --basedir=/usr --user=mysql
mysql 2070 1950 0 06:28 ? 00:00:01 /usr/libexec/mysqld --basedir=/usr --datadir=/var/lib/mysql --user=mysql --log-error=/var/log/mysqld.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock
cloudera 7650 7486 0 07:03 pts/0 00:00:00 grep mysql
[cloudera@quickstart ~]$
[cloudera@quickstart ~]$
[cloudera@quickstart ~]$ sqoop import-all-tables \
> -m 1 \
> --connect jdbc:mysql://quickstart:3306/retail_db \
> --username=retail_dba \
> --password=cloudera \
> --compression-codec=snappy \
> --as-parquetfile \
> --warehouse-dir=/user/hive/warehouse \
> --hive-import
Warning: /usr/lib/sqoop/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
16/06/01 07:03:30 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6-cdh5.7.0
16/06/01 07:03:30 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
16/06/01 07:03:30 INFO tool.BaseSqoopTool: Using Hive-specific delimiters for output. You can override
16/06/01 07:03:30 INFO tool.BaseSqoopTool: delimiters with --fields-terminated-by, etc.
16/06/01 07:03:30 WARN tool.BaseSqoopTool: It seems that you're doing hive import directly into default
16/06/01 07:03:30 WARN tool.BaseSqoopTool: hive warehouse directory which is not supported. Sqoop is
16/06/01 07:03:30 WARN tool.BaseSqoopTool: firstly importing data into separate directory and then
16/06/01 07:03:30 WARN tool.BaseSqoopTool: inserting data into hive. Please consider removing
16/06/01 07:03:30 WARN tool.BaseSqoopTool: --target-dir or --warehouse-dir into /user/hive/warehouse in
16/06/01 07:03:30 WARN tool.BaseSqoopTool: case that you will detect any issues.
16/06/01 07:03:31 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
16/06/01 07:03:31 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:03:31 INFO tool.CodeGenTool: Will generate java class as codegen_categories
16/06/01 07:03:31 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `categories` AS t LIMIT 1
16/06/01 07:03:31 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `categories` AS t LIMIT 1
16/06/01 07:03:31 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:03:33 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:03:33 WARN manager.MySQLManager: It looks like you are importing from mysql.
16/06/01 07:03:33 WARN manager.MySQLManager: This transfer can be faster! Use the --direct
16/06/01 07:03:33 WARN manager.MySQLManager: option to exercise a MySQL-specific fast path.
16/06/01 07:03:33 INFO manager.MySQLManager: Setting zero DATETIME behavior to convertToNull (mysql)
16/06/01 07:03:33 INFO mapreduce.ImportJobBase: Beginning import of categories
16/06/01 07:03:33 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:03:33 INFO Configuration.deprecation: mapred.jar is deprecated. Instead, use mapreduce.job.jar
16/06/01 07:03:34 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `categories` AS t LIMIT 1
16/06/01 07:03:34 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `categories` AS t LIMIT 1
16/06/01 07:03:35 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:03:35 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:03:35 INFO hive.metastore: Connected to metastore.
16/06/01 07:03:35 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:03:35 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:03:35 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:03:35 INFO hive.metastore: Connected to metastore.
16/06/01 07:03:36 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:03:36 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:03:36 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:03:36 INFO hive.metastore: Connected to metastore.
16/06/01 07:03:36 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:03:36 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:03:36 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:03:36 INFO hive.metastore: Connected to metastore.
16/06/01 07:03:36 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: categories
16/06/01 07:03:37 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:03:37 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:03:37 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:03:37 INFO hive.metastore: Connected to metastore.
16/06/01 07:03:49 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps
16/06/01 07:03:49 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:03:59 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:03:59 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:03:59 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0001
16/06/01 07:04:00 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0001
16/06/01 07:04:00 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0001/
16/06/01 07:04:00 INFO mapreduce.Job: Running job: job_1464787780964_0001
16/06/01 07:04:27 INFO mapreduce.Job: Job job_1464787780964_0001 running in uber mode : false
16/06/01 07:04:27 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:04:43 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:04:44 INFO mapreduce.Job: Job job_1464787780964_0001 completed successfully
16/06/01 07:04:44 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=211514
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=5685
HDFS: Number of bytes written=3446
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=12540
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=12540
Total vcore-seconds taken by all map tasks=12540
Total megabyte-seconds taken by all map tasks=12840960
Map-Reduce Framework
Map input records=58
Map output records=58
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=141
CPU time spent (ms)=2640
Physical memory (bytes) snapshot=336764928
Virtual memory (bytes) snapshot=1592012800
Total committed heap usage (bytes)=326107136
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:04:44 INFO mapreduce.ImportJobBase: Transferred 3.3652 KB in 55.2648 seconds (62.3543 bytes/sec)
16/06/01 07:04:44 INFO mapreduce.ImportJobBase: Retrieved 58 records.
16/06/01 07:04:44 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:04:44 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `customers` AS t LIMIT 1
16/06/01 07:04:44 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:04:49 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:04:49 INFO mapreduce.ImportJobBase: Beginning import of customers
16/06/01 07:04:49 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:04:49 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `customers` AS t LIMIT 1
16/06/01 07:04:49 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `customers` AS t LIMIT 1
16/06/01 07:04:49 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:04:49 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:04:49 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:04:49 INFO hive.metastore: Connected to metastore.
16/06/01 07:04:50 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:04:50 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:04:50 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:04:50 INFO hive.metastore: Connected to metastore.
16/06/01 07:04:50 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:04:50 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:04:50 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:04:50 INFO hive.metastore: Connected to metastore.
16/06/01 07:04:50 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: customers
16/06/01 07:04:50 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:04:50 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:04:50 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:04:50 INFO hive.metastore: Connected to metastore.
16/06/01 07:04:50 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:05:00 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:05:00 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:05:00 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0002
16/06/01 07:05:00 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0002
16/06/01 07:05:00 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0002/
16/06/01 07:05:00 INFO mapreduce.Job: Running job: job_1464787780964_0002
16/06/01 07:05:27 INFO mapreduce.Job: Job job_1464787780964_0002 running in uber mode : false
16/06/01 07:05:27 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:05:44 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:05:49 INFO mapreduce.Job: Job job_1464787780964_0002 completed successfully
16/06/01 07:05:49 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=212281
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=13005
HDFS: Number of bytes written=257967
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=13696
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=13696
Total vcore-seconds taken by all map tasks=13696
Total megabyte-seconds taken by all map tasks=14024704
Map-Reduce Framework
Map input records=12435
Map output records=12435
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=148
CPU time spent (ms)=5080
Physical memory (bytes) snapshot=341532672
Virtual memory (bytes) snapshot=1585790976
Total committed heap usage (bytes)=268435456
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:05:49 INFO mapreduce.ImportJobBase: Transferred 251.9209 KB in 59.1842 seconds (4.2566 KB/sec)
16/06/01 07:05:49 INFO mapreduce.ImportJobBase: Retrieved 12435 records.
16/06/01 07:05:49 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:05:49 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `departments` AS t LIMIT 1
16/06/01 07:05:50 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:05:55 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:05:55 INFO mapreduce.ImportJobBase: Beginning import of departments
16/06/01 07:05:55 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:05:55 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `departments` AS t LIMIT 1
16/06/01 07:05:55 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `departments` AS t LIMIT 1
16/06/01 07:05:55 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:05:55 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:05:55 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:05:55 INFO hive.metastore: Connected to metastore.
16/06/01 07:05:55 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:05:55 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:05:56 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:05:56 INFO hive.metastore: Connected to metastore.
16/06/01 07:05:56 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:05:56 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:05:56 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:05:56 INFO hive.metastore: Connected to metastore.
16/06/01 07:05:56 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: departments
16/06/01 07:05:56 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:05:56 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:05:56 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:05:56 INFO hive.metastore: Connected to metastore.
16/06/01 07:05:56 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:06:06 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:06:06 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:06:06 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0003
16/06/01 07:06:06 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0003
16/06/01 07:06:06 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0003/
16/06/01 07:06:06 INFO mapreduce.Job: Running job: job_1464787780964_0003
16/06/01 07:06:35 INFO mapreduce.Job: Job job_1464787780964_0003 running in uber mode : false
16/06/01 07:06:35 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:08:20 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:08:33 INFO mapreduce.Job: Job job_1464787780964_0003 completed successfully
16/06/01 07:08:34 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=211387
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=4453
HDFS: Number of bytes written=1982
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=105084
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=105084
Total vcore-seconds taken by all map tasks=105084
Total megabyte-seconds taken by all map tasks=107606016
Map-Reduce Framework
Map input records=6
Map output records=6
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=1788
CPU time spent (ms)=3220
Physical memory (bytes) snapshot=253460480
Virtual memory (bytes) snapshot=1580556288
Total committed heap usage (bytes)=267911168
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:08:34 INFO mapreduce.ImportJobBase: Transferred 1.9355 KB in 157.3586 seconds (12.5954 bytes/sec)
16/06/01 07:08:34 INFO mapreduce.ImportJobBase: Retrieved 6 records.
16/06/01 07:08:34 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:08:34 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `order_items` AS t LIMIT 1
16/06/01 07:08:34 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:08:39 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:08:39 INFO mapreduce.ImportJobBase: Beginning import of order_items
16/06/01 07:08:39 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:08:39 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `order_items` AS t LIMIT 1
16/06/01 07:08:39 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `order_items` AS t LIMIT 1
16/06/01 07:08:39 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:08:39 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:08:39 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:08:39 INFO hive.metastore: Connected to metastore.
16/06/01 07:08:40 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:08:40 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:08:40 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:08:40 INFO hive.metastore: Connected to metastore.
16/06/01 07:08:40 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:08:40 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:08:40 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:08:40 INFO hive.metastore: Connected to metastore.
16/06/01 07:08:40 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: order_items
16/06/01 07:08:40 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:08:40 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:08:40 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:08:40 INFO hive.metastore: Connected to metastore.
16/06/01 07:08:40 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:08:49 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:08:49 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:08:49 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0004
16/06/01 07:08:49 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0004
16/06/01 07:08:49 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0004/
16/06/01 07:08:49 INFO mapreduce.Job: Running job: job_1464787780964_0004
16/06/01 07:09:23 INFO mapreduce.Job: Job job_1464787780964_0004 running in uber mode : false
16/06/01 07:09:23 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:09:56 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:10:20 INFO mapreduce.Job: Job job_1464787780964_0004 completed successfully
16/06/01 07:10:21 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=211981
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=9725
HDFS: Number of bytes written=1649632
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=46491
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=46491
Total vcore-seconds taken by all map tasks=46491
Total megabyte-seconds taken by all map tasks=47606784
Map-Reduce Framework
Map input records=172198
Map output records=172198
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=741
CPU time spent (ms)=6910
Physical memory (bytes) snapshot=412913664
Virtual memory (bytes) snapshot=1575092224
Total committed heap usage (bytes)=342360064
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:10:21 INFO mapreduce.ImportJobBase: Transferred 1.5732 MB in 100.2474 seconds (16.0699 KB/sec)
16/06/01 07:10:21 INFO mapreduce.ImportJobBase: Retrieved 172198 records.
16/06/01 07:10:21 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:10:21 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `orders` AS t LIMIT 1
16/06/01 07:10:21 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:10:27 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:10:27 INFO mapreduce.ImportJobBase: Beginning import of orders
16/06/01 07:10:27 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:10:27 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `orders` AS t LIMIT 1
16/06/01 07:10:27 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `orders` AS t LIMIT 1
16/06/01 07:10:27 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:10:27 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:10:27 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:10:27 INFO hive.metastore: Connected to metastore.
16/06/01 07:10:27 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:10:27 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:10:27 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:10:27 INFO hive.metastore: Connected to metastore.
16/06/01 07:10:28 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:10:28 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:10:28 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:10:28 INFO hive.metastore: Connected to metastore.
16/06/01 07:10:28 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: orders
16/06/01 07:10:28 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:10:28 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:10:28 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:10:28 INFO hive.metastore: Connected to metastore.
16/06/01 07:10:28 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:10:37 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:10:37 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:10:37 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0005
16/06/01 07:10:38 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0005
16/06/01 07:10:38 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0005/
16/06/01 07:10:38 INFO mapreduce.Job: Running job: job_1464787780964_0005
16/06/01 07:11:10 INFO mapreduce.Job: Job job_1464787780964_0005 running in uber mode : false
16/06/01 07:11:10 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:11:33 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:11:45 INFO mapreduce.Job: Job job_1464787780964_0005 completed successfully
16/06/01 07:11:46 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=211574
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=6589
HDFS: Number of bytes written=489972
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=23779
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=23779
Total vcore-seconds taken by all map tasks=23779
Total megabyte-seconds taken by all map tasks=24349696
Map-Reduce Framework
Map input records=68883
Map output records=68883
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=780
CPU time spent (ms)=6320
Physical memory (bytes) snapshot=398024704
Virtual memory (bytes) snapshot=1588559872
Total committed heap usage (bytes)=342360064
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:11:46 INFO mapreduce.ImportJobBase: Transferred 478.4883 KB in 77.3825 seconds (6.1834 KB/sec)
16/06/01 07:11:46 INFO mapreduce.ImportJobBase: Retrieved 68883 records.
16/06/01 07:11:46 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:11:46 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `products` AS t LIMIT 1
16/06/01 07:11:46 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:11:53 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:11:53 INFO mapreduce.ImportJobBase: Beginning import of products
16/06/01 07:11:53 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:11:53 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `products` AS t LIMIT 1
16/06/01 07:11:53 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `products` AS t LIMIT 1
16/06/01 07:11:54 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:11:54 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:11:54 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:11:54 INFO hive.metastore: Connected to metastore.
16/06/01 07:11:54 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:11:54 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:11:54 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:11:54 INFO hive.metastore: Connected to metastore.
16/06/01 07:11:54 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:11:54 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:11:54 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:11:54 INFO hive.metastore: Connected to metastore.
16/06/01 07:11:54 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: products
16/06/01 07:11:54 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:11:54 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:11:54 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:11:54 INFO hive.metastore: Connected to metastore.
16/06/01 07:11:55 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:12:04 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:12:04 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:12:04 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0006
16/06/01 07:12:04 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0006
16/06/01 07:12:04 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0006/
16/06/01 07:12:04 INFO mapreduce.Job: Running job: job_1464787780964_0006
16/06/01 07:12:39 INFO mapreduce.Job: Job job_1464787780964_0006 running in uber mode : false
16/06/01 07:12:39 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:13:14 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:13:15 INFO mapreduce.Job: Job job_1464787780964_0006 completed successfully
16/06/01 07:13:15 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=211879
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=9261
HDFS: Number of bytes written=47239
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=22970
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=22970
Total vcore-seconds taken by all map tasks=22970
Total megabyte-seconds taken by all map tasks=23521280
Map-Reduce Framework
Map input records=1345
Map output records=1345
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=119
CPU time spent (ms)=3040
Physical memory (bytes) snapshot=298315776
Virtual memory (bytes) snapshot=1590517760
Total committed heap usage (bytes)=185597952
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:13:15 INFO mapreduce.ImportJobBase: Transferred 46.1318 KB in 80.5216 seconds (586.6626 bytes/sec)
16/06/01 07:13:15 INFO mapreduce.ImportJobBase: Retrieved 1345 records.
-m 1 \
--connect jdbc:mysql://quickstart:3306/retail_db \
--username=retail_dba \
--password=cloudera \
--compression-codec=snappy \
--as-parquetfile \
--warehouse-dir=/user/hive/warehouse \
--hive-import
=========================================================
[cloudera@quickstart ~]$ ps -ef|grep mysql
root 1950 1 0 06:28 ? 00:00:00 /bin/sh /usr/bin/mysqld_safe --datadir=/var/lib/mysql --socket=/var/lib/mysql/mysql.sock --pid-file=/var/run/mysqld/mysqld.pid --basedir=/usr --user=mysql
mysql 2070 1950 0 06:28 ? 00:00:01 /usr/libexec/mysqld --basedir=/usr --datadir=/var/lib/mysql --user=mysql --log-error=/var/log/mysqld.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock
cloudera 7650 7486 0 07:03 pts/0 00:00:00 grep mysql
[cloudera@quickstart ~]$
[cloudera@quickstart ~]$
[cloudera@quickstart ~]$ sqoop import-all-tables \
> -m 1 \
> --connect jdbc:mysql://quickstart:3306/retail_db \
> --username=retail_dba \
> --password=cloudera \
> --compression-codec=snappy \
> --as-parquetfile \
> --warehouse-dir=/user/hive/warehouse \
> --hive-import
Warning: /usr/lib/sqoop/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
16/06/01 07:03:30 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6-cdh5.7.0
16/06/01 07:03:30 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
16/06/01 07:03:30 INFO tool.BaseSqoopTool: Using Hive-specific delimiters for output. You can override
16/06/01 07:03:30 INFO tool.BaseSqoopTool: delimiters with --fields-terminated-by, etc.
16/06/01 07:03:30 WARN tool.BaseSqoopTool: It seems that you're doing hive import directly into default
16/06/01 07:03:30 WARN tool.BaseSqoopTool: hive warehouse directory which is not supported. Sqoop is
16/06/01 07:03:30 WARN tool.BaseSqoopTool: firstly importing data into separate directory and then
16/06/01 07:03:30 WARN tool.BaseSqoopTool: inserting data into hive. Please consider removing
16/06/01 07:03:30 WARN tool.BaseSqoopTool: --target-dir or --warehouse-dir into /user/hive/warehouse in
16/06/01 07:03:30 WARN tool.BaseSqoopTool: case that you will detect any issues.
16/06/01 07:03:31 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
16/06/01 07:03:31 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:03:31 INFO tool.CodeGenTool: Will generate java class as codegen_categories
16/06/01 07:03:31 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `categories` AS t LIMIT 1
16/06/01 07:03:31 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `categories` AS t LIMIT 1
16/06/01 07:03:31 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:03:33 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:03:33 WARN manager.MySQLManager: It looks like you are importing from mysql.
16/06/01 07:03:33 WARN manager.MySQLManager: This transfer can be faster! Use the --direct
16/06/01 07:03:33 WARN manager.MySQLManager: option to exercise a MySQL-specific fast path.
16/06/01 07:03:33 INFO manager.MySQLManager: Setting zero DATETIME behavior to convertToNull (mysql)
16/06/01 07:03:33 INFO mapreduce.ImportJobBase: Beginning import of categories
16/06/01 07:03:33 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:03:33 INFO Configuration.deprecation: mapred.jar is deprecated. Instead, use mapreduce.job.jar
16/06/01 07:03:34 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `categories` AS t LIMIT 1
16/06/01 07:03:34 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `categories` AS t LIMIT 1
16/06/01 07:03:35 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:03:35 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:03:35 INFO hive.metastore: Connected to metastore.
16/06/01 07:03:35 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:03:35 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:03:35 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:03:35 INFO hive.metastore: Connected to metastore.
16/06/01 07:03:36 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:03:36 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:03:36 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:03:36 INFO hive.metastore: Connected to metastore.
16/06/01 07:03:36 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:03:36 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:03:36 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:03:36 INFO hive.metastore: Connected to metastore.
16/06/01 07:03:36 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: categories
16/06/01 07:03:37 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:03:37 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:03:37 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:03:37 INFO hive.metastore: Connected to metastore.
16/06/01 07:03:49 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps
16/06/01 07:03:49 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:03:59 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:03:59 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:03:59 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0001
16/06/01 07:04:00 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0001
16/06/01 07:04:00 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0001/
16/06/01 07:04:00 INFO mapreduce.Job: Running job: job_1464787780964_0001
16/06/01 07:04:27 INFO mapreduce.Job: Job job_1464787780964_0001 running in uber mode : false
16/06/01 07:04:27 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:04:43 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:04:44 INFO mapreduce.Job: Job job_1464787780964_0001 completed successfully
16/06/01 07:04:44 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=211514
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=5685
HDFS: Number of bytes written=3446
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=12540
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=12540
Total vcore-seconds taken by all map tasks=12540
Total megabyte-seconds taken by all map tasks=12840960
Map-Reduce Framework
Map input records=58
Map output records=58
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=141
CPU time spent (ms)=2640
Physical memory (bytes) snapshot=336764928
Virtual memory (bytes) snapshot=1592012800
Total committed heap usage (bytes)=326107136
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:04:44 INFO mapreduce.ImportJobBase: Transferred 3.3652 KB in 55.2648 seconds (62.3543 bytes/sec)
16/06/01 07:04:44 INFO mapreduce.ImportJobBase: Retrieved 58 records.
16/06/01 07:04:44 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:04:44 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `customers` AS t LIMIT 1
16/06/01 07:04:44 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:04:49 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:04:49 INFO mapreduce.ImportJobBase: Beginning import of customers
16/06/01 07:04:49 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:04:49 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `customers` AS t LIMIT 1
16/06/01 07:04:49 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `customers` AS t LIMIT 1
16/06/01 07:04:49 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:04:49 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:04:49 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:04:49 INFO hive.metastore: Connected to metastore.
16/06/01 07:04:50 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:04:50 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:04:50 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:04:50 INFO hive.metastore: Connected to metastore.
16/06/01 07:04:50 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:04:50 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:04:50 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:04:50 INFO hive.metastore: Connected to metastore.
16/06/01 07:04:50 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: customers
16/06/01 07:04:50 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:04:50 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:04:50 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:04:50 INFO hive.metastore: Connected to metastore.
16/06/01 07:04:50 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:05:00 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:05:00 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:05:00 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0002
16/06/01 07:05:00 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0002
16/06/01 07:05:00 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0002/
16/06/01 07:05:00 INFO mapreduce.Job: Running job: job_1464787780964_0002
16/06/01 07:05:27 INFO mapreduce.Job: Job job_1464787780964_0002 running in uber mode : false
16/06/01 07:05:27 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:05:44 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:05:49 INFO mapreduce.Job: Job job_1464787780964_0002 completed successfully
16/06/01 07:05:49 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=212281
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=13005
HDFS: Number of bytes written=257967
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=13696
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=13696
Total vcore-seconds taken by all map tasks=13696
Total megabyte-seconds taken by all map tasks=14024704
Map-Reduce Framework
Map input records=12435
Map output records=12435
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=148
CPU time spent (ms)=5080
Physical memory (bytes) snapshot=341532672
Virtual memory (bytes) snapshot=1585790976
Total committed heap usage (bytes)=268435456
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:05:49 INFO mapreduce.ImportJobBase: Transferred 251.9209 KB in 59.1842 seconds (4.2566 KB/sec)
16/06/01 07:05:49 INFO mapreduce.ImportJobBase: Retrieved 12435 records.
16/06/01 07:05:49 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:05:49 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `departments` AS t LIMIT 1
16/06/01 07:05:50 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:05:55 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:05:55 INFO mapreduce.ImportJobBase: Beginning import of departments
16/06/01 07:05:55 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:05:55 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `departments` AS t LIMIT 1
16/06/01 07:05:55 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `departments` AS t LIMIT 1
16/06/01 07:05:55 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:05:55 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:05:55 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:05:55 INFO hive.metastore: Connected to metastore.
16/06/01 07:05:55 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:05:55 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:05:56 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:05:56 INFO hive.metastore: Connected to metastore.
16/06/01 07:05:56 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:05:56 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:05:56 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:05:56 INFO hive.metastore: Connected to metastore.
16/06/01 07:05:56 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: departments
16/06/01 07:05:56 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:05:56 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:05:56 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:05:56 INFO hive.metastore: Connected to metastore.
16/06/01 07:05:56 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:06:06 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:06:06 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:06:06 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0003
16/06/01 07:06:06 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0003
16/06/01 07:06:06 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0003/
16/06/01 07:06:06 INFO mapreduce.Job: Running job: job_1464787780964_0003
16/06/01 07:06:35 INFO mapreduce.Job: Job job_1464787780964_0003 running in uber mode : false
16/06/01 07:06:35 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:08:20 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:08:33 INFO mapreduce.Job: Job job_1464787780964_0003 completed successfully
16/06/01 07:08:34 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=211387
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=4453
HDFS: Number of bytes written=1982
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=105084
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=105084
Total vcore-seconds taken by all map tasks=105084
Total megabyte-seconds taken by all map tasks=107606016
Map-Reduce Framework
Map input records=6
Map output records=6
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=1788
CPU time spent (ms)=3220
Physical memory (bytes) snapshot=253460480
Virtual memory (bytes) snapshot=1580556288
Total committed heap usage (bytes)=267911168
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:08:34 INFO mapreduce.ImportJobBase: Transferred 1.9355 KB in 157.3586 seconds (12.5954 bytes/sec)
16/06/01 07:08:34 INFO mapreduce.ImportJobBase: Retrieved 6 records.
16/06/01 07:08:34 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:08:34 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `order_items` AS t LIMIT 1
16/06/01 07:08:34 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:08:39 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:08:39 INFO mapreduce.ImportJobBase: Beginning import of order_items
16/06/01 07:08:39 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:08:39 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `order_items` AS t LIMIT 1
16/06/01 07:08:39 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `order_items` AS t LIMIT 1
16/06/01 07:08:39 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:08:39 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:08:39 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:08:39 INFO hive.metastore: Connected to metastore.
16/06/01 07:08:40 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:08:40 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:08:40 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:08:40 INFO hive.metastore: Connected to metastore.
16/06/01 07:08:40 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:08:40 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:08:40 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:08:40 INFO hive.metastore: Connected to metastore.
16/06/01 07:08:40 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: order_items
16/06/01 07:08:40 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:08:40 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:08:40 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:08:40 INFO hive.metastore: Connected to metastore.
16/06/01 07:08:40 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:08:49 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:08:49 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:08:49 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0004
16/06/01 07:08:49 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0004
16/06/01 07:08:49 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0004/
16/06/01 07:08:49 INFO mapreduce.Job: Running job: job_1464787780964_0004
16/06/01 07:09:23 INFO mapreduce.Job: Job job_1464787780964_0004 running in uber mode : false
16/06/01 07:09:23 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:09:56 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:10:20 INFO mapreduce.Job: Job job_1464787780964_0004 completed successfully
16/06/01 07:10:21 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=211981
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=9725
HDFS: Number of bytes written=1649632
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=46491
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=46491
Total vcore-seconds taken by all map tasks=46491
Total megabyte-seconds taken by all map tasks=47606784
Map-Reduce Framework
Map input records=172198
Map output records=172198
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=741
CPU time spent (ms)=6910
Physical memory (bytes) snapshot=412913664
Virtual memory (bytes) snapshot=1575092224
Total committed heap usage (bytes)=342360064
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:10:21 INFO mapreduce.ImportJobBase: Transferred 1.5732 MB in 100.2474 seconds (16.0699 KB/sec)
16/06/01 07:10:21 INFO mapreduce.ImportJobBase: Retrieved 172198 records.
16/06/01 07:10:21 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:10:21 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `orders` AS t LIMIT 1
16/06/01 07:10:21 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:10:27 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:10:27 INFO mapreduce.ImportJobBase: Beginning import of orders
16/06/01 07:10:27 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:10:27 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `orders` AS t LIMIT 1
16/06/01 07:10:27 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `orders` AS t LIMIT 1
16/06/01 07:10:27 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:10:27 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:10:27 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:10:27 INFO hive.metastore: Connected to metastore.
16/06/01 07:10:27 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:10:27 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:10:27 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:10:27 INFO hive.metastore: Connected to metastore.
16/06/01 07:10:28 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:10:28 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:10:28 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:10:28 INFO hive.metastore: Connected to metastore.
16/06/01 07:10:28 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: orders
16/06/01 07:10:28 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:10:28 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:10:28 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:10:28 INFO hive.metastore: Connected to metastore.
16/06/01 07:10:28 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:10:37 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:10:37 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:10:37 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0005
16/06/01 07:10:38 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0005
16/06/01 07:10:38 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0005/
16/06/01 07:10:38 INFO mapreduce.Job: Running job: job_1464787780964_0005
16/06/01 07:11:10 INFO mapreduce.Job: Job job_1464787780964_0005 running in uber mode : false
16/06/01 07:11:10 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:11:33 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:11:45 INFO mapreduce.Job: Job job_1464787780964_0005 completed successfully
16/06/01 07:11:46 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=211574
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=6589
HDFS: Number of bytes written=489972
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=23779
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=23779
Total vcore-seconds taken by all map tasks=23779
Total megabyte-seconds taken by all map tasks=24349696
Map-Reduce Framework
Map input records=68883
Map output records=68883
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=780
CPU time spent (ms)=6320
Physical memory (bytes) snapshot=398024704
Virtual memory (bytes) snapshot=1588559872
Total committed heap usage (bytes)=342360064
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:11:46 INFO mapreduce.ImportJobBase: Transferred 478.4883 KB in 77.3825 seconds (6.1834 KB/sec)
16/06/01 07:11:46 INFO mapreduce.ImportJobBase: Retrieved 68883 records.
16/06/01 07:11:46 INFO tool.CodeGenTool: Beginning code generation
16/06/01 07:11:46 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `products` AS t LIMIT 1
16/06/01 07:11:46 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/lib/hadoop-mapreduce
Note: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
16/06/01 07:11:53 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-cloudera/compile/9c5d81b8a4fc931d02a97efaa7fd3d8a/codegen_categories.jar
16/06/01 07:11:53 INFO mapreduce.ImportJobBase: Beginning import of products
16/06/01 07:11:53 INFO Configuration.deprecation: mapred.job.tracker is deprecated. Instead, use mapreduce.jobtracker.address
16/06/01 07:11:53 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `products` AS t LIMIT 1
16/06/01 07:11:53 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `products` AS t LIMIT 1
16/06/01 07:11:54 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:11:54 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:11:54 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:11:54 INFO hive.metastore: Connected to metastore.
16/06/01 07:11:54 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:11:54 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:11:54 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:11:54 INFO hive.metastore: Connected to metastore.
16/06/01 07:11:54 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:11:54 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:11:54 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:11:54 INFO hive.metastore: Connected to metastore.
16/06/01 07:11:54 INFO hive.HiveManagedMetadataProvider: Creating a managed Hive table named: products
16/06/01 07:11:54 INFO hive.metastore: Closed a connection to metastore, current connections: 0
16/06/01 07:11:54 INFO hive.metastore: Trying to connect to metastore with URI thrift://127.0.0.1:9083
16/06/01 07:11:54 INFO hive.metastore: Opened a connection to metastore, current connections: 1
16/06/01 07:11:54 INFO hive.metastore: Connected to metastore.
16/06/01 07:11:55 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
16/06/01 07:12:04 INFO db.DBInputFormat: Using read commited transaction isolation
16/06/01 07:12:04 INFO mapreduce.JobSubmitter: number of splits:1
16/06/01 07:12:04 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1464787780964_0006
16/06/01 07:12:04 INFO impl.YarnClientImpl: Submitted application application_1464787780964_0006
16/06/01 07:12:04 INFO mapreduce.Job: The url to track the job: http://quickstart.cloudera:8088/proxy/application_1464787780964_0006/
16/06/01 07:12:04 INFO mapreduce.Job: Running job: job_1464787780964_0006
16/06/01 07:12:39 INFO mapreduce.Job: Job job_1464787780964_0006 running in uber mode : false
16/06/01 07:12:39 INFO mapreduce.Job: map 0% reduce 0%
16/06/01 07:13:14 INFO mapreduce.Job: map 100% reduce 0%
16/06/01 07:13:15 INFO mapreduce.Job: Job job_1464787780964_0006 completed successfully
16/06/01 07:13:15 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=211879
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=9261
HDFS: Number of bytes written=47239
HDFS: Number of read operations=48
HDFS: Number of large read operations=0
HDFS: Number of write operations=10
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=22970
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=22970
Total vcore-seconds taken by all map tasks=22970
Total megabyte-seconds taken by all map tasks=23521280
Map-Reduce Framework
Map input records=1345
Map output records=1345
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=119
CPU time spent (ms)=3040
Physical memory (bytes) snapshot=298315776
Virtual memory (bytes) snapshot=1590517760
Total committed heap usage (bytes)=185597952
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=0
16/06/01 07:13:15 INFO mapreduce.ImportJobBase: Transferred 46.1318 KB in 80.5216 seconds (586.6626 bytes/sec)
16/06/01 07:13:15 INFO mapreduce.ImportJobBase: Retrieved 1345 records.
Subscribe to:
Posts (Atom)
