|
php可以通过thrift连接hbase，同样php可以通过thrift读取hadoop资源(hdfs资源)。 准备： php需要thrift的libary packages:hadoop-0.20.2\src\contrib\thriftfs\gen-php 源码： <?php $globals['thrift_root'] = rootpath . '/lib/thrift'; require_once($globals['thrift_root'].'/thrift.php'); require_once($globals['thrift_root'].'/transport/tsocket.php'); require_once($globals['thrift_root'].'/transport/tbufferedtransport.php'); require_once($globals['thrift_root'].'/protocol/tbinaryprotocol.php'); require_once($globals["thrift_root"] . "/packages/hadoopfs/thrifthadoopfilesystem.php"); $hadoop_socket = new tsocket("localhost", 59256); $hadoop_socket -> setsendtimeout(10000); // ten seconds $hadoop_socket -> setrecvtimeout(20000); // twenty seconds $hadoop_transport = new tbufferedtransport($hadoop_socket); $hadoop_protocol = new tbinaryprotocol($hadoop_transport); $hadoopclient = new thrifthadoopfilesystemclient($hadoop_protocol); $hadoop_transport -> open(); try { // create directory $dirpathname = new hadoopfs_pathname(array("pathname" => "/user/root/hadoop")); if($hadoopclient -> exists($dirpathname) == true) { echo $dirpathname -> pathname . " exists.\n"; } else { $result = $hadoopclient -> mkdirs($dirpathname); } // put file $filepathname = new hadoopfs_pathname(array("pathname" => $dirpathname -> pathname . "/hello.txt")); $localfile = fopen("hello.txt", "rb"); $hdfsfile = $hadoopclient -> create($filepathname); while(true) { $data = fread($localfile, 1024); if(strlen($data) == 0) break; $hadoopclient -> write($hdfsfile, $data); } $hadoopclient -> close($hdfsfile); fclose($localfile); // get file echo "read file:\n"; print_r($filepathname); $data = ""; $hdfsfile = $hadoopclient -> open($filepathname); print_r($hdfsfile); while(true) { $data = $hadoopclient -> read($hdfsfile, 0, 1024); if(strlen($data) == 0) break; print $data; } $hadoopclient -> close($hdfsfile); echo "liststatus:\n"; $result = $hadoopclient -> liststatus($dirpathname); print_r($result); foreach($result as $key => $value) { if($value -> isdir == "1") print "dir\t"; else print "file\t"; print $value -> block_replication . "\t" . $value -> length . "\t" . $value -> modification_time . "\t" . $value -> permission . "\t" . $value -> owner . "\t" . $value -> group . "\t" . $value -> path . "\n"; } $hadoop_transport -> close(); } catch(exception $e) { print_r($e); } ?> 启动hadoop的thrift hadoop-0.20.2\src\contrib\thriftfs\scripts\start_thrift_server.sh 59256 problem one: 在系统目录创建文件，而不是在hadoop目录中创建文件 原因： thrift启动时加载默认的配置文件 解决方法： 修改start_thrift_server.sh文件 top=/usr/local/hadoop-0.20.2 classpath=$classpath:$top/conf problem two: java.lang.nullpointerexception at org.apache.hadoop.thriftfs.hadoopthriftserver$hadoopthrifthandler.write(hadoopthriftserver.java:282) at org.apache.hadoop.thriftfs.api.thrifthadoopfilesystem$processor$write.process(unknown source) at org.apache.hadoop.thriftfs.api.thrifthadoopfilesystem$processor.process(unknown source) at com.facebook.thrift.server.tthreadpoolserver$workerprocess.run(unknown source) at java.util.concurrent.threadpoolexecutor$worker.runtask(threadpoolexecutor.java:886) at java.util.concurrent.threadpoolexecutor$worker.run(threadpoolexecutor.java:908) at java.lang.thread.run(thread.java:662) 原因： java返回的map hash id为long类型，而php(32位)无法存储long类型的数据，导致转换成float数据后丢失精度。 private long nextid = new random().nextlong(); java返回数据：4207488029786584864 |
