First of all, we need to build a Hadoop server
Please refer to: Hadoop 3.2.0 fully distributed cluster building and wordcount running
Create a new Maven project and add Hadoop related jar package
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>3.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>3.2.0</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.2</version>
</dependency>
Write HDFS tool class
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.net.URI;
/**
* @author LionLi
* @date 2019-2-7
*/
public class HDFSUtils {
/** hdfs Server Address */
private static final String hdfsUrl = "hdfs://192.168.0.200:8020/";
/** Access users */
private static final String username = "root";
public static void main(String[] args) throws Exception {
String filePath = "/user/root/demo.txt";
StringBuilder sb = new StringBuilder();
for (int i = 1; i < 100001; i++) {
sb.append("hadoop demo " + i + "\r\n");
}
HDFSCreateAndWriteFileValue(hdfsUrl,username, filePath,sb.toString());
HDFSReadFileValue(hdfsUrl,username, filePath);
}
/**
* Read the contents of the file
* @param url server address
* @param user access user
* @param filePath file path
*/
private static void HDFSReadFileValue(
String url,String user, String filePath) throws Exception {
Configuration conf = new Configuration();
try (FileSystem fileSystem = FileSystem.get(new URI(url), conf, user)) {
Path demo = new Path(filePath);
if (fileSystem.exists(demo)) {
System.out.println("/user/root/demo.txt file exist");
FSDataInputStream fsdis = fileSystem.open(demo);
byte[] bytes = new byte[fsdis.available()];
fsdis.readFully(bytes);
System.out.println(new String(bytes));
} else {
System.out.println("/user/root/demo.txt file not exist");
}
}
}
/**
* Create and write the contents of the file
* @param url server address
* @param user Access user
* @param filePath The path to the file to be created
* @param value The value to be written
*/
private static void HDFSCreateAndWriteFileValue(
String url,String user,String filePath,String value) throws Exception {
Configuration conf = new Configuration();
try (FileSystem fileSystem = FileSystem.get(new URI(url), conf, user)) {
Path demo = new Path(filePath);
if (fileSystem.exists(demo)) {
System.out.println("/user/root/demo.txt file exist");
if (fileSystem.delete(demo, false)) {
if (fileSystem.exists(demo)) {
System.out.println("/user/root/demo.txt failed to delete file");
} else {
System.out.println("/user/root/demo.txt success to delete file");
FSDataOutputStream fsdos = fileSystem.create(demo);
fsdos.write(value.getBytes());
}
}
} else {
System.out.println("/user/root/demo.txt file not exist");
FSDataOutputStream fsdos = fileSystem.create(demo);
fsdos.write(value.getBytes());
}
}
}
}
test run
The test was successful
Read More:
- Read and write BMP image with Pure C language
- [Solved] ERROR: error loading sources list: The read operation timed out
- [Nginx] solution: it can’t be accessed on the background API interface after HTTPS (access the specified port through the domain name)
- [Solved] sys.stderr.write(f“ERROR: {exc}“) SyntaxError: invalid syntax
- Termux setting path environment variable
- [Solved] NIC cannot be generated vf, intel/mellanox, write error: Cannot allocate memory “not enough MMIO resources for SR-IOV”
- [Solved] fatal error: cannot write PCH file: required memory segment unavailable
- Ubuntu found an error during apt operation [How to Solve]
- Anaconda prompt running environment.yml [How to Solve]
- [Solved] Cmake: symbol lookup error: cmake: undefined symbol: archive_write_add_filter_zstd
- [SSH error] ssh_exchange_identification: read: Connection reset by peer
- Nginx Error: nginx: [emerg] the “ssl“ parameter requires ngx_http_ssl_module in /project/api/nginx.conf:
- [Solved] E xx: xxxx:xx ERR thread 0x536753929472 setscheduler failed Operation not permitted
- CDH operation and maintenance: child node cloudera SCM agent starts Error
- Configure the node environment under Linux, internal/modules/cjs/loader.js:583 throw err; solution
- [Solved] Environment: failed conda all InvalidVersionSpecError: Invalid version spec: =2.7
- [Solved] Syntax Error: TypeError: Cannot read property ‘parseComponent‘ of undefined (vue报错)
- [Solved] Linux — 9 — txt files are copied from windows to Linux and read error
- [Solved] mosquitto log Error: SSL routines:SSL3_READ_BYTES:tlsv1 alert decrypt error
- [Solved] Linux error: SSH_exchange_identification: read: Connection reset by peer