hive metastore - kudu - impala 연동

hive-site.xml

<property>
<name>hive.metastore.transactional.event.listeners</name>
<value>
org.apache.hive.hcatalog.listener.DbNotificationListener,
org.apache.kudu.hive.metastore.KuduMetastorePlugin
</value>
</property>

<property>
<name>hive.metastore.disallow.incompatible.col.type.changes</name>
<value>false</value>
</property> <property>
<name>hive.metastore.notifications.add.thrift.objects</name>
<value>true</value>
</property>

kudu master start option add

--hive_metastore_uris=thrift:/test.co.kr:9083 user add

hdfs dfs -mkdir /user/impala
hdfs dfs -chown -R impala:hdfs /user/impala drwxr-xr-x - impala hdfs 0 2022-04-05 16:59 /user/impala

set acl

hdfs dfs -setfacl -m default:user:impala:rwx hdfs:///warehouse/tablespace/managed/hive
hdfs dfs -setfacl -m user:impala:rwx hdfs:///warehouse/tablespace/managed/hive
hdfs dfs -setfacl -m default:user:impala:rwx hdfs:///warehouse/tablespace/external/hive

hdfs dfs -getfacl hdfs:///warehouse/tablespace/external/hive
# file: hdfs:///warehouse/tablespace/external/hive
# owner: hive
# group: hadoop
# flags: --t
user::rwx
group::rwx
other::rwx
default:user::rwx
default:user:hive:rwx
default:user:impala:rwx
default:group::rwx
default:mask::rwx
default:other::rwx
hdfs dfs -getfacl hdfs:///warehouse/tablespace/managed/hive
# file: hdfs:///warehouse/tablespace/managed/hive
# owner: hive
# group: hadoop
user::rwx
group::---
other::---
default:user::rwx
default:user:hive:rwx
default:user:impala:rwx
default:group::---
default:mask::rwx
default:other::---
impala start option add

/etc/default/impala
KUDU_MASTER_HOSTS=test.co.kr:7051 추가

---TEST---
$ hive
hive> CREATE DATABASE new_db_from_hive;
hive> CREATE TABLE new_db_from_hive.new_table_from_hive (x INT);
hive> quit;

$ impala-shell
> REFRESH new_db_from_hive.new_table_from_hive;
ERROR: AnalysisException: Database does not exist: new_db_from_hive

> INVALIDATE METADATA new_db_from_hive.new_table_from_hive; > SHOW DATABASES LIKE 'new*';
+--------------------+
| new_db_from_hive |
+--------------------+ > SHOW TABLES IN new_db_from_hive;
+---------------------+
| new_table_from_hive |
+---------------------+
Advanced hive-interactive-site
hive.strict.managed.tables
false
hive-site
hive.strict.managed.tables
false
hive-site, hivemetastore-site 차이?ㅋ




KUDU_MASTER_HOSTS=test.co.kr:7051

IMPALA_CATALOG_ARGS=" -log_dir=${IMPALA_LOG_DIR} -webserver_doc_root=${IMPALA_WEB_ROOT} -kudu_master_hosts=${KUDU_MASTER_HOSTS}"
IMPALA_STATE_STORE_ARGS=" -log_dir=${IMPALA_LOG_DIR} -state_store_port=${IMPALA_STATE_STORE_PORT} -webserver_doc_root=${IMPALA_WEB_ROOT} -kudu_master_hosts=${KUDU_MASTER_HOSTS}"
IMPALA_SERVER_ARGS=" \
-log_dir=${IMPALA_LOG_DIR} \
-catalog_service_host=${IMPALA_CATALOG_SERVICE_HOST} \
-state_store_port=${IMPALA_STATE_STORE_PORT} \
-use_statestore \
-state_store_host=${IMPALA_STATE_STORE_HOST} \
-be_port=${IMPALA_BACKEND_PORT} \
-webserver_doc_root=${IMPALA_WEB_ROOT} \
-kudu_master_hosts=${KUDU_MASTER_HOSTS}"


kudu/sbin/kudu-master --rpc_bind_addresses=0.0.0.0:7051 --log-dir=kudu/logs/master --fs_wal_dir=/var/log/kudu_master --fs_data_dirs=/var/log/kudu_master --webserver_doc_root=kudu/www --webserver_port=8051 --master_addresses=hostname:7051 --hive_metastore_uris=thrift://hostname:9083