{"id":1648,"date":"2011-05-17T08:10:45","date_gmt":"2011-05-17T08:10:45","guid":{"rendered":"http:\/\/www.msinterdev.org\/wp\/2011\/05\/17\/pig-cassandra-hadoop-%ec%84%a4%ec%a0%95\/"},"modified":"2011-05-17T08:10:45","modified_gmt":"2011-05-17T08:10:45","slug":"pig-cassandra-hadoop-%ec%84%a4%ec%a0%95","status":"publish","type":"post","link":"http:\/\/www.msinterdev.org\/wp\/2011\/05\/17\/pig-cassandra-hadoop-%ec%84%a4%ec%a0%95\/","title":{"rendered":"PIG Cassandra HADOOP \uc124\uc815"},"content":{"rendered":"

PIG\ub97c \ud1b5\ud574 \uce74\uc0b0\ub4dc\ub77c(Cassandra) \uc640 \uc5f0\uacb0\uc5d0 \uc131\uacf5\ud55c \uc124\uc815\uc785\ub2c8\ub2e4. <\/p>\n

==========================================================================================
core-site.xml<\/span> on HADOOP<\/span>
==========================================================================================
<configuration>
  <property>
    <name>fs.default.name<\/name>
    <value>hdfs:\/\/localhost:54310<\/value>
  <\/property><\/p>\n

  <property>
     <name>hadoop.tmp.dir<\/name>
     <value>\/usr\/lib64\/hadoop\/tmp<\/value>
  <\/property>
  <property>
      <name>cassandra.thrift.address<\/name>
      <value>localhost<\/value>
  <\/property>
  <property>
      <name>cassandra.thrift.port<\/name>
      <value>9160<\/value>
  <\/property>
  <property>
      <name>cassandra.partitioner.class<\/name>
      <value>org.apache.cassandra.dht.RandomPartitioner<\/value>
  <\/property>
<\/configuration>
==========================================================================================<\/p>\n

==========================================================================================
system environment(\/etc\/profile<\/span>) for HADDOP, PIG and CASSANDRA<\/span>
==========================================================================================
export JAVA_HOME=\/usr\/lib64\/jvm\/java-6-sun<\/p>\n

export HADOOP_HOME=\/usr\/lib64\/hadoop
export HADOOPDIR=$HADOOP_HOME\/conf
export HADOOP_CONF_DIR=$HADOOPDIR
export HADOOP_CONF=$HADOOPDIR
export HADOOPDIR=$HADOOP_CONF
export HADOOP_CLASSPATH=$HADOOP_CONF:$CASSANDRA_LIB<\/p>\n

export PIG_HOME=\/usr\/lib64\/pig
export PIG_CLASSPATH=$HADOOPDIR
export PIG_CONF=$HADOOPDIR
export PIG_CONF_DIR=$HADOOPDIR
export PIG_RPC_PORT=9160
export PIG_INITIAL_ADDRESS=localhost<\/p>\n

export cassandra_home=\/usr\/share\/cassandra
export CASSANDRA_HOME=$cassandra_home
export CASSANDRA_LIB=\/usr\/share\/cassandra
export CASSANDRA_CLASSPATH=$cassandra_home<\/p>\n

export PATH=$PATH:$HADOOP_HOME\/bin:$PIG_HOME\/bin
================================================================================================<\/p>\n

================================================================================================
HADOOP_CLASSPATH<\/span>
================================================================================================<\/p>\n

One configuration note on getting the task trackers to be able to perform queries over Cassandra: you’ll want to update your HADOOP_CLASSPATH<\/tt> in your <hadoop>\/conf\/hadoop-env.sh<\/tt> to include the Cassandra lib libraries. For example you’ll want to do something like this in the hadoop-env.sh<\/tt> on each of your task trackers: <\/p>\n

export HADOOP_CLASSPATH=\/opt\/cassandra\/lib\/*:$HADOOP_CLASSPATH<\/pre>\n
================================================================================================<\/p>\n
PIG\ub97c \uc0ac\uc6a9\ud558\uc9c0 \uc54a\uace0  HADOOP\uc5d0\uc11c CASSANDRA\ub97c \uc5f0\uacb0\ud558\uae30 \uc704\ud574\uc11c\ub294 \uba87 \uac00\uc9c0 \uc124\uc815\uc774 \ud544\uc694\ud558\uc9c0 \uc548\uc2b5\ub2c8\ub2e4.
PIG \uc5d0\uc11c \uc9c1\uc811 Cassandra \uc5d0\uc11c \uc811\uadfc\ud558\uc9c0 \uc54a\uc744 \uacbd\uc6b0 Java\uc5d0\uc11c \ub4dc\ub77c\uc774\ubc84\ub97c \uc774\uc6a9\ud574\uc11c \uc9c1\uc811 \uc811\uadfc \ud560 \uc218 \uc788\uc73c\ub9ac\ub77c \ud310\ub2e8 \ub429\ub2c8\ub2e4. <\/p>\n
<\/p>\n","protected":false},"excerpt":{"rendered":"
PIG\ub97c \ud1b5\ud574 \uce74\uc0b0\ub4dc\ub77c(Cassandra) \uc640 \uc5f0\uacb0\uc5d0 \uc131\uacf5\ud55c \uc124\uc815\uc785\ub2c8\ub2e4. ==========================================================================================core-site.xml on HADOOP==========================================================================================<configuration>  <property>    <name>fs.default.name<\/name>    <value>hdfs:\/\/localhost:54310<\/value>  <\/property>   <property>     <name>hadoop.tmp.dir<\/name>     <value>\/usr\/lib64\/hadoop\/tmp<\/value>  <\/property>  <property>      <name>cassandra.thrift.address<\/name>      <value>localhost<\/value>  <\/property>  <property>      <name>cassandra.thrift.port<\/name>      <value>9160<\/value>  <\/property>  <property>      <name>cassandra.partitioner.class<\/name>      <value>org.apache.cassandra.dht.RandomPartitioner<\/value>  <\/property><\/configuration>========================================================================================== ==========================================================================================system environment(\/etc\/profile) for HADDOP, PIG and CASSANDRA==========================================================================================export JAVA_HOME=\/usr\/lib64\/jvm\/java-6-sun […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[2948],"tags":[2979,2980,2981,2926,2982,2983,844,2984,2985,2927,2986,2987,2988],"class_list":["post-1648","post","type-post","status-publish","format-standard","hentry","category-about-knowledge-system_apache_hadoop","tag-etc-profile","tag-cassandra","tag-core-site-xml","tag-hadoop","tag-pig","tag-profile","tag-844","tag-2984","tag-2985","tag-2927","tag-2986","tag-2987","tag-2988"],"_links":{"self":[{"href":"http:\/\/www.msinterdev.org\/wp\/wp-json\/wp\/v2\/posts\/1648","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/www.msinterdev.org\/wp\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.msinterdev.org\/wp\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.msinterdev.org\/wp\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.msinterdev.org\/wp\/wp-json\/wp\/v2\/comments?post=1648"}],"version-history":[{"count":0,"href":"http:\/\/www.msinterdev.org\/wp\/wp-json\/wp\/v2\/posts\/1648\/revisions"}],"wp:attachment":[{"href":"http:\/\/www.msinterdev.org\/wp\/wp-json\/wp\/v2\/media?parent=1648"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.msinterdev.org\/wp\/wp-json\/wp\/v2\/categories?post=1648"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.msinterdev.org\/wp\/wp-json\/wp\/v2\/tags?post=1648"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}