#Before user can write to hdfs we need to set permissions
sudo -u hdfs hadoop fs -mkdir /user/username
sudo -u hdfs hadoop fs -chown username: usergroup /user/
username
#if we need to write to hdfs from remote machine ( append avro file, for example )
//if user exist it return its ugi
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("username");
ugi.doAs(new PrivilegedExceptionAction() {
public Void run() throws Exception {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://hadoop-server");
FileSystem fs = FileSystem.get(conf);
DatumWriter xmlOrderInfoWriter = new SpecificDatumWriter(XmlOrderInfo.class);
DataFileWriter dataFileWriter = new DataFileWriter(xmlOrderInfoWriter);
Path filePath = new Path("/user//data/myfolder/test-xml-files.avro");
OutputStream out = fs.append(filePath);
dataFileWriter.appendTo(new FsInput(filePath,conf),out);
dataFileWriter.append(xmlOrderInfo);
dataFileWriter.close();
out.close();
System.out.println("Test avro file is appended in HDFS successfully");
return null;
}
});
sudo -u hdfs hadoop fs -mkdir /user/username
sudo -u hdfs hadoop fs -chown username
username
#if we need to write to hdfs from remote machine ( append avro file, for example )
//if user exist it return its ugi
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("username
ugi.doAs(new PrivilegedExceptionAction
public Void run() throws Exception {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://hadoop-server");
FileSystem fs = FileSystem.get(conf);
DatumWriter
DataFileWriter
Path filePath = new Path("/user/
OutputStream out = fs.append(filePath);
dataFileWriter.appendTo(new FsInput(filePath,conf),out);
dataFileWriter.append(xmlOrderInfo);
dataFileWriter.close();
out.close();
System.out.println("Test avro file is appended in HDFS successfully");
return null;
}
});
Комментариев нет:
Отправить комментарий