HADOOP-19549. Add -o option in hdfs "count" command to show the owner's summarization.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -1921,20 +1921,45 @@ public long getLength(Path f) throws IOException {
    * @return content summary.
    */
   public ContentSummary getContentSummary(Path f) throws IOException {
+    return getContentSummary(f, null);
+  }
+
+  /** Return the {@link ContentSummary} of a given {@link Path}.
+   * @param f path to use
+   * @param owner path owner
+   * @throws FileNotFoundException if the path does not resolve
+   * @throws IOException IO failure
+   * @return content summary.
+   */
+  public ContentSummary getContentSummary(Path f, String owner) throws IOException {
     FileStatus status = getFileStatus(f);
     if (status.isFile()) {
       // f is a file
       long length = status.getLen();
-      return new ContentSummary.Builder().length(length).
-          fileCount(1).directoryCount(0).spaceConsumed(length).build();
+      // owner mismatch, output 0 0 0
+      if (owner == null || owner.equals(status.getOwner())) {
+        return new ContentSummary.Builder().length(length).
+            fileCount(1).directoryCount(0).spaceConsumed(length).build();
+      } else {
+        return new ContentSummary.Builder().length(length).
+            fileCount(0).directoryCount(0).spaceConsumed(0).build();
+      }
     }
     // f is a directory
-    long[] summary = {0, 0, 1};
+    long i = (owner == null || owner.equals(status.getOwner())) ? 1 : 0;
+    long[] summary = {0, 0, i};
     for(FileStatus s : listStatus(f)) {
       long length = s.getLen();
-      ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) :
-          new ContentSummary.Builder().length(length).
-          fileCount(1).directoryCount(0).spaceConsumed(length).build();
+      ContentSummary c;
+      if (s.isDirectory()) {
+        c = getContentSummary(s.getPath(), owner);
+      } else if (owner == null || owner.equals(s.getOwner())) {
+        c = new ContentSummary.Builder().length(length).
+            fileCount(1).directoryCount(0).spaceConsumed(length).build();
+      } else {
+        // skip owner mismatch
+        continue;
+      }
       summary[0] += c.getLength();
       summary[1] += c.getFileCount();
       summary[2] += c.getDirectoryCount();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java

@@ -57,14 +57,15 @@ public static void registerCommands(CommandFactory factory) {
   private static final String OPTION_QUOTA_AND_USAGE = "u";
   private static final String OPTION_ECPOLICY = "e";
   private static final String OPTION_SNAPSHOT_COUNT = "s";
+  private static final String OPTION_OWNER = "o";
 
   public static final String NAME = "count";
   public static final String USAGE =
       "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
           + "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
           OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
           + "] [-" + OPTION_ECPOLICY + "] [-" + OPTION_SNAPSHOT_COUNT
-          + "] <path> ...";
+          + "] [-" + OPTION_OWNER + " [<user>]] <path> ...";
   public static final String DESCRIPTION =
       "Count the number of directories, files and bytes under the paths\n" +
           "that match the specified file pattern.  The output columns are:\n" +
@@ -92,10 +93,11 @@ public static void registerCommands(CommandFactory factory) {
           "ram_disk, ssd, disk, archive and nvdimm.\n" +
           "It can also pass the value '', 'all' or 'ALL' to specify all " +
           "the storage types.\n" +
-          "The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and \n" +
-          "the usage against the quota without the detailed content summary."+
-          "The -" + OPTION_ECPOLICY + " option shows the erasure coding policy."
-          + "The -" + OPTION_SNAPSHOT_COUNT + " option shows snapshot counts.";
+          "The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and " +
+          "the usage against the quota without the detailed content summary. \n" +
+          "The -" + OPTION_ECPOLICY + " option shows the erasure coding policy. \n" +
+          "The -" + OPTION_SNAPSHOT_COUNT + " option shows snapshot counts. \n" +
+          "The -" + OPTION_OWNER + " option shows the owner's counts.";
 
   private boolean showQuotas;
   private boolean humanReadable;
@@ -105,6 +107,7 @@ public static void registerCommands(CommandFactory factory) {
   private boolean excludeSnapshots;
   private boolean displayECPolicy;
   private boolean showSnapshot;
+  private String owner;
 
   /** Constructor */
   public Count() {}
@@ -128,6 +131,7 @@ protected void processOptions(LinkedList<String> args) {
         OPTION_EXCLUDE_SNAPSHOT,
         OPTION_ECPOLICY, OPTION_SNAPSHOT_COUNT);
     cf.addOptionWithValue(OPTION_TYPE);
+    cf.addOptionWithValue(OPTION_OWNER);
     cf.parse(args);
     if (args.isEmpty()) { // default path is the current working directory
       args.add(".");
@@ -138,6 +142,7 @@ protected void processOptions(LinkedList<String> args) {
     excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
     displayECPolicy = cf.getOpt(OPTION_ECPOLICY);
     showSnapshot = cf.getOpt(OPTION_SNAPSHOT_COUNT);
+    owner = cf.getOptValue(OPTION_OWNER);
 
     if (showQuotas || showQuotasAndUsageOnly) {
       String types = cf.getOptValue(OPTION_TYPE);
@@ -157,7 +162,10 @@ protected void processOptions(LinkedList<String> args) {
 
     if (cf.getOpt(OPTION_HEADER)) {
       StringBuilder headString = new StringBuilder();
-      if (showQuotabyType) {
+      if (owner != null && !owner.isEmpty()) {
+        headString.append("OWNER");
+        headString.append(ContentSummary.getHeader(false));
+      } else if (showQuotabyType) {
         headString.append(QuotaUsage.getStorageTypeHeader(storageTypes));
       } else {
         if (showQuotasAndUsageOnly) {
@@ -195,7 +203,12 @@ private List<StorageType> getAndCheckStorageTypes(String types) {
   @Override
   protected void processPath(PathData src) throws IOException {
     StringBuilder outputString = new StringBuilder();
-    if (showQuotasAndUsageOnly || showQuotabyType) {
+    if (owner != null && !owner.isEmpty()) {
+      outputString.append(owner);
+      ContentSummary summary = src.fs.getContentSummary(src.path, owner);
+      outputString.append(summary.toString(
+          false, isHumanReadable()));
+    } else if (showQuotasAndUsageOnly || showQuotabyType) {
       QuotaUsage usage = src.fs.getQuotaUsage(src.path);
       outputString.append(usage.toString(
           isHumanReadable(), showQuotabyType, storageTypes));

hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md

@@ -139,7 +139,7 @@ Identical to the -get command.
 count
 -----
 
-Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] [-e] [-s] <paths> `
+Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] [-e] [-s] [-o [<user>]] <paths> `
 
 Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
 
@@ -165,6 +165,8 @@ The ERASURECODING\_POLICY is name of the policy for the file. If an erasure codi
 
 The -s option shows the snapshot counts for each directory.
 
+The -o option shows the owner's counts.
+
 Example:
 
 * `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
@@ -176,6 +178,7 @@ Example:
 * `hadoop fs -count -u -h -v hdfs://nn1.example.com/file1`
 * `hadoop fs -count -e hdfs://nn1.example.com/file1`
 * `hadoop fs -count -s hdfs://nn1.example.com/file1`
+* `hadoop fs -count -o user hdfs://nn1.example.com/file1`
 
 Exit Code:
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java

@@ -141,6 +141,7 @@ public Token<?>[] addDelegationTokens(String renewer, Credentials creds)
     public String getScheme();
     public Path fixRelativePart(Path p);
     public ContentSummary getContentSummary(Path f);
+    public ContentSummary getContentSummary(Path f, String owner);
     public QuotaUsage getQuotaUsage(Path f);
     void setQuota(Path f, long namespaceQuota, long storagespaceQuota);
     void setQuotaByStorageType(Path f, StorageType type, long quota);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java

@@ -116,6 +116,7 @@ public FSDataOutputStream create(Path f, FsPermission permission,
     public short getReplication(Path src);
     public void processDeleteOnExit();
     public ContentSummary getContentSummary(Path f);
+    public ContentSummary getContentSummary(Path f, String owner);
     public QuotaUsage getQuotaUsage(Path f);
     void setQuota(Path f, long namespaceQuota, long storagespaceQuota);
     void setQuotaByStorageType(Path f, StorageType type, long quota);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java

@@ -439,6 +439,27 @@ public void processPathWithSnapshotHeader() throws Exception {
     verifyNoMoreInteractions(out);
   }
 
+  @Test
+  public void processOptionsHeaderWithOwner() {
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("-v");
+    options.add("-o");
+    options.add("user");
+    options.add("dummy");
+
+    PrintStream out = mock(PrintStream.class);
+
+    Count count = new Count();
+    count.out = out;
+
+    count.processOptions(options);
+
+    String withOwnerHeader =
+        "OWNER   DIR_COUNT   FILE_COUNT       CONTENT_SIZE PATHNAME";
+    verify(out).println(withOwnerHeader);
+    verifyNoMoreInteractions(out);
+  }
+
   @Test
   public void getCommandName() {
     Count count = new Count();
@@ -477,7 +498,7 @@ public void getUsage() {
     String actual = count.getUsage();
     String expected =
         "-count [-q] [-h] [-v] [-t [<storage type>]]"
-        + " [-u] [-x] [-e] [-s] <path> ...";
+        + " [-u] [-x] [-e] [-s] [-o [<user>]] <path> ...";
     assertEquals(expected, actual, "Count.getUsage");
   }
 
@@ -507,10 +528,12 @@ public void getDescription() {
         + "ram_disk, ssd, disk, archive and nvdimm.\n"
         + "It can also pass the value '', 'all' or 'ALL' to specify all the "
         + "storage types.\n"
-        + "The -u option shows the quota and \n"
-        + "the usage against the quota without the detailed content summary."
-        + "The -e option shows the erasure coding policy."
-        + "The -s option shows snapshot counts.";
+        + "The -u option shows the quota and "
+        + "the usage against the quota without the detailed content summary. \n"
+        + "The -e option shows the erasure coding policy. \n"
+        + "The -s option shows snapshot counts. \n"
+        + "The -o option shows the owner's counts.";
+
 
     assertEquals(expected, actual, "Count.getDescription");
   }

hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

@@ -250,7 +250,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] \[-x\] \[-e\] \[-s\] &lt;path&gt; \.\.\. :( )*</expected-output>
+          <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] \[-x\] \[-e\] \[-s\] \[-o \[&lt;user&gt;\]\] &lt;path&gt; \.\.\. :( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>