From ff1514217438eef1e23f5b20ccc25bdbc4a73a26 Mon Sep 17 00:00:00 2001 From: pccmast Date: Mon, 7 Dec 2020 16:44:39 +0800 Subject: [PATCH] test faults --- .../java/com/github/hcsp/multithread/1.txt | 3 + .../java/com/github/hcsp/multithread/2.txt | 3 + .../java/com/github/hcsp/multithread/3.txt | 3 + .../java/com/github/hcsp/multithread/4.txt | 3 + .../java/com/github/hcsp/multithread/5.txt | 3 + .../java/com/github/hcsp/multithread/6.txt | 3 + .../multithread/MultiThreadWordCount1.java | 68 ++++++++++++++- .../multithread/MultiThreadWordCount2.java | 83 ++++++++++++++++++- 8 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 src/main/java/com/github/hcsp/multithread/1.txt create mode 100644 src/main/java/com/github/hcsp/multithread/2.txt create mode 100644 src/main/java/com/github/hcsp/multithread/3.txt create mode 100644 src/main/java/com/github/hcsp/multithread/4.txt create mode 100644 src/main/java/com/github/hcsp/multithread/5.txt create mode 100644 src/main/java/com/github/hcsp/multithread/6.txt diff --git a/src/main/java/com/github/hcsp/multithread/1.txt b/src/main/java/com/github/hcsp/multithread/1.txt new file mode 100644 index 0000000..ee98119 --- /dev/null +++ b/src/main/java/com/github/hcsp/multithread/1.txt @@ -0,0 +1,3 @@ +I am a teacher +I am a person +1 \ No newline at end of file diff --git a/src/main/java/com/github/hcsp/multithread/2.txt b/src/main/java/com/github/hcsp/multithread/2.txt new file mode 100644 index 0000000..1540a45 --- /dev/null +++ b/src/main/java/com/github/hcsp/multithread/2.txt @@ -0,0 +1,3 @@ +I am a teacher +I am a person +2 \ No newline at end of file diff --git a/src/main/java/com/github/hcsp/multithread/3.txt b/src/main/java/com/github/hcsp/multithread/3.txt new file mode 100644 index 0000000..2820c76 --- /dev/null +++ b/src/main/java/com/github/hcsp/multithread/3.txt @@ -0,0 +1,3 @@ +I am a teacher +I am a person +3 \ No newline at end of file diff --git a/src/main/java/com/github/hcsp/multithread/4.txt b/src/main/java/com/github/hcsp/multithread/4.txt new file mode 100644 index 0000000..a08a52c --- /dev/null +++ b/src/main/java/com/github/hcsp/multithread/4.txt @@ -0,0 +1,3 @@ +I am a teacher +I am a person +4 \ No newline at end of file diff --git a/src/main/java/com/github/hcsp/multithread/5.txt b/src/main/java/com/github/hcsp/multithread/5.txt new file mode 100644 index 0000000..705b9bd --- /dev/null +++ b/src/main/java/com/github/hcsp/multithread/5.txt @@ -0,0 +1,3 @@ +I am a teacher +I am a person +5 \ No newline at end of file diff --git a/src/main/java/com/github/hcsp/multithread/6.txt b/src/main/java/com/github/hcsp/multithread/6.txt new file mode 100644 index 0000000..824c25f --- /dev/null +++ b/src/main/java/com/github/hcsp/multithread/6.txt @@ -0,0 +1,3 @@ +I am a teacher +I am a person +6 \ No newline at end of file diff --git a/src/main/java/com/github/hcsp/multithread/MultiThreadWordCount1.java b/src/main/java/com/github/hcsp/multithread/MultiThreadWordCount1.java index 15c4aae..2617eb4 100644 --- a/src/main/java/com/github/hcsp/multithread/MultiThreadWordCount1.java +++ b/src/main/java/com/github/hcsp/multithread/MultiThreadWordCount1.java @@ -1,12 +1,76 @@ package com.github.hcsp.multithread; +import java.io.BufferedReader; import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.*; public class MultiThreadWordCount1 { // 使用threadNum个线程，并发统计文件中各单词的数量 - public static Map count(int threadNum, List files) { - return null; + // 使用线程池 + public static Map count(int threadNum, List files) throws FileNotFoundException, ExecutionException, InterruptedException { + List>> resultsFuture = new ArrayList<>(); + ExecutorService threadPool = Executors.newFixedThreadPool(threadNum); + for (File file : files) { + BufferedReader bufferedReader = new BufferedReader(new FileReader(file)); + Future> mapFuture = threadPool.submit(new readFileCallable(bufferedReader)); + resultsFuture.add(mapFuture); + } + threadPool.shutdown(); + return mergeCountResults(resultsFuture); + } + + private static Map mergeCountResults(List>> resultsFuture) throws ExecutionException, InterruptedException { + Map finalResults = new HashMap<>(); + for (Future> mapFuture : resultsFuture) { + Map stringIntegerMap = mapFuture.get(); + for (Map.Entry stringIntegerEntry : stringIntegerMap.entrySet()) { + String word = stringIntegerEntry.getKey(); + int updatedValue = finalResults.getOrDefault(word, 0) + stringIntegerEntry.getValue(); + finalResults.put(word, updatedValue); + } + } + return finalResults; + } + + public static class readFileCallable implements Callable> { + BufferedReader bufferedReader; + + public readFileCallable(BufferedReader bufferedReader) { + this.bufferedReader = bufferedReader; + } + + @Override + public Map call() throws Exception { + Map fileWords = new HashMap<>(); + String line; + while ((line = bufferedReader.readLine()) != null) { + String[] words = line.split(" "); + for (String word : words) { + fileWords.put(word, fileWords.getOrDefault(word, 0) + 1); + } + } + return fileWords; + } + } + + public static void main(String[] args) throws FileNotFoundException, ExecutionException, InterruptedException { + List files = new ArrayList<>(); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\1.txt")); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\2.txt")); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\3.txt")); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\4.txt")); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\5.txt")); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\6.txt")); + Map results = MultiThreadWordCount1.count(5, files); + for (Map.Entry resultEntry : results.entrySet()) { + System.out.println("Key: " + resultEntry.getKey()+" Value: " + resultEntry.getValue()); + } } } + diff --git a/src/main/java/com/github/hcsp/multithread/MultiThreadWordCount2.java b/src/main/java/com/github/hcsp/multithread/MultiThreadWordCount2.java index 3f23afa..c9a3e62 100644 --- a/src/main/java/com/github/hcsp/multithread/MultiThreadWordCount2.java +++ b/src/main/java/com/github/hcsp/multithread/MultiThreadWordCount2.java @@ -1,8 +1,85 @@ package com.github.hcsp.multithread; +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinTask; +import java.util.concurrent.RecursiveTask; + public class MultiThreadWordCount2 { // 使用threadNum个线程，并发统计文件中各单词的数量 - // public static Map count(int threadNum, List files) { - // return null; - // } + // 使用ForkJoinPool + public static Map count(int threadNum, List files) { + ForkJoinPool forkJoinPool = new ForkJoinPool(threadNum); + ForkJoinTask> task = new readFileWordTask(files); + return forkJoinPool.invoke(task); + } + + public static class readFileWordTask extends RecursiveTask> { + private static final int THRESHOLD = 5; + List files; + + public readFileWordTask(List files) { + this.files = files; + } + + @Override + protected Map compute() { + Map results = null; + if (files.size() < THRESHOLD) { + for (File file : files) { + try { + BufferedReader bufferedReader = new BufferedReader(new FileReader(file)); + results = new ConcurrentHashMap<>(); + String line; + while ((line = bufferedReader.readLine()) != null) { + String[] words = line.split(" "); + for (String word : words) { + results.put(word, results.getOrDefault(word, 0) + 1); + } + } + System.out.println(file.getCanonicalFile()); + } catch (IOException e) { + e.printStackTrace(); + } + } + } else { + List subFiles1 = files.subList(0, files.size() / 2); + List subFiles2 = files.subList(files.size() / 2, files.size()); + readFileWordTask subTask1 = new readFileWordTask(subFiles1); + readFileWordTask subTask2 = new readFileWordTask(subFiles2); + invokeAll(subTask1, subTask2); + Map subResult1 = subTask1.join(); + Map subResult2 = subTask2.join(); + results = mergeTwoMap(subResult1, subResult2); + } + return results; + } + + private Map mergeTwoMap(Map subResult1, Map subResult2) { + Map results = new ConcurrentHashMap<>(subResult1); + for (Map.Entry stringIntegerEntry : subResult2.entrySet()) { + String word = stringIntegerEntry.getKey(); + results.put(word, subResult2.getOrDefault(word, 0) + 1); + } + return results; + } + } + + public static void main(String[] args) throws IOException { + List files = new ArrayList<>(); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\1.txt")); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\2.txt")); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\3.txt")); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\4.txt")); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\5.txt")); + files.add(new File("D:\\project\\multithread-word-count\\src\\main\\java\\com\\github\\hcsp\\multithread\\6.txt")); + Map results = MultiThreadWordCount2.count(5, files); + for (Map.Entry resultEntry : results.entrySet()) { + System.out.println("Key: " + resultEntry.getKey()+" Value: " + resultEntry.getValue()); + } + } }