Changes after review: Add repository url and name to the output. Rena…

…me columns of repository output to standardize with files output. Add pulling files by repositories option to GitLab which was missing.
r-world-devs · Dec 5, 2023 · d0c5888 · d0c5888
1 parent 9faa730
commit d0c5888
Show file tree

Hide file tree

Showing 26 changed files with 143 additions and 146 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -10,6 +10,10 @@
 
 - fixed pulling responses when GitLab groups have private or empty content ([#314](https://github.com/r-world-devs/GitStats/issues/314).
 
+## Minor changes:
+
+- rename column names for repository output - `id` to `repo_id` and `name` to `repo_name`.
+
 # GitStats 1.0.0
 
 ## Breaking changes:

diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R
@@ -108,7 +108,7 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
         org = org,
         settings = list(search_param = "org")
       )
-      repos_names <- repos_table$name
+      repos_names <- repos_table$repo_name
 
       if (settings$search_param == "org") {
         if (!private$scan_all) {
@@ -263,7 +263,7 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
           repo <- data.frame(repo) %>%
             dplyr::relocate(
               default_branch,
-              .after = name
+              .after = repo_name
             )
         })
       } else {
@@ -471,10 +471,10 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
           from = "org",
           org = org
         )
-        repositories <- purrr::map(repos_list, ~ .$name)
+        repositories <- purrr::map(repos_list, ~ .$repo_name)
         def_branches <- purrr::map(repos_list, ~ .$default_branch$name)
       } else {
-        repositories <- pulled_repos$name
+        repositories <- pulled_repos$repo_name
         def_branches <- pulled_repos$default_branch
       }
       files_list <- purrr::map(file_path, function(file_path) {
@@ -507,12 +507,13 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
         files_table <- purrr::map(file_path, function(file) {
           purrr::imap(files_response[[file]], function(repository, name) {
             data.frame(
-              "repository_name" = repository$name,
-              "repository_id" = repository$id,
+              "repo_name" = repository$name,
+              "repo_id" = repository$id,
               "organization" = org,
               "file_path" = file,
               "file_content" = repository$object$text,
               "file_size" = repository$object$byteSize,
+              "repo_url" = repository$url,
               "api_url" = self$gql_api_url
             )
           }) %>%

diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R
@@ -191,7 +191,7 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab",
            ) %>%
            dplyr::relocate(
              default_branch,
-             .after = name
+             .after = repo_name
            )
        } else {
          repos_table <- NULL
@@ -305,12 +305,13 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab",
        if (!is.null(files_response)) {
          files_table <- purrr::map(files_response, function(project) {
            data.frame(
-             "repository_name" = project$name,
-             "repository_id" = project$id,
+             "repo_name" = project$name,
+             "repo_id" = project$id,
              "organization" = org,
              "file_path" = project$repository$blobs$nodes[[1]]$name,
              "file_content" = project$repository$blobs$nodes[[1]]$rawBlob,
              "file_size" = as.integer(project$repository$blobs$nodes[[1]]$size),
+             "repo_url" = project$webUrl,
              "api_url" = self$gql_api_url
            )
          }) %>%

diff --git a/R/EngineRest.R b/R/EngineRest.R
@@ -107,7 +107,7 @@ EngineRest <- R6::R6Class("EngineRest",
 
       if (length(repos_dt) > 0) {
         repos_dt <- dplyr::mutate(repos_dt,
-          id = as.character(id),
+          repo_id = as.character(repo_id),
           created_at = as.POSIXct(created_at),
           last_activity_at = as.POSIXct(last_activity_at),
           forks = as.integer(forks),

diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R
@@ -61,9 +61,9 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
     #' @param settings A list of  `GitStats` settings.
     #' @return A table of commits.
     pull_commits = function(org,
-                           date_from,
-                           date_until = Sys.date(),
-                           settings) {
+                            date_from,
+                            date_until = Sys.date(),
+                            settings) {
       NULL
     },
 
@@ -116,7 +116,7 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
         if (!private$scan_all) {
           cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{unique(repos_table$organization)}] Pulling contributors...")
         }
-        repo_iterator <- paste0(repos_table$organization, "/", repos_table$name)
+        repo_iterator <- paste0(repos_table$organization, "/", repos_table$repo_name)
         user_name <- rlang::expr(.$login)
         repos_table$contributors <- purrr::map_chr(repo_iterator, function(repos_id) {
           contributors_endpoint <- paste0(self$rest_api_url, "/repos/", repos_id, "/contributors")
@@ -229,31 +229,25 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
                                byte_max) {
       if (total_n >= 0 & total_n < 1e3) {
         resp_list <- list()
-
         for (page in 1:(total_n %/% 100)) {
           resp_list <- self$response(
             paste0(search_endpoint, "+size:0..", byte_max, "&page=", page, "&per_page=100")
           )[["items"]] %>%
             append(resp_list, .)
         }
-
         resp_list
       } else if (total_n >= 1e3) {
         resp_list <- list()
         index <- c(0, 50)
-
         spinner <- cli::make_spinner(
           which = "timeTravel",
           template = cli::col_grey(
             "GitHub search limit (1000 results) exceeded. Results will be divided. {spin}"
           )
         )
-
         while (index[2] < as.numeric(byte_max)) {
           size_formula <- paste0("+size:", as.character(index[1]), "..", as.character(index[2]))
-
           spinner$spin()
-
           n_count <- tryCatch(
             {
               self$response(paste0(search_endpoint, size_formula))[["total_count"]]
@@ -262,7 +256,6 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
               NULL
             }
           )
-
           if (is.null(n_count)) {
             NULL
           } else if ((n_count - 1) %/% 100 > 0) {
@@ -273,9 +266,7 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
             resp_list <- self$response(paste0(search_endpoint, size_formula, "&page=1&per_page=100"))[["items"]] %>%
               append(resp_list, .)
           }
-
           index[1] <- index[2]
-
           if (index[2] < 1e3) {
             index[2] <- index[2] + 50
           }
@@ -300,8 +291,8 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
     tailor_repos_info = function(repos_list) {
       repos_list <- purrr::map(repos_list, function(repo) {
         list(
-          "id" = repo$id,
-          "name" = repo$name,
+          "repo_id" = repo$id,
+          "repo_name" = repo$name,
           "default_branch" = repo$default_branch,
           "stars" = repo$stargazers_count,
           "forks" = repo$forks_count,
@@ -322,7 +313,7 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
     # @return A table of repositories with added information on issues.
     pull_repos_issues = function(repos_table) {
       if (nrow(repos_table) > 0) {
-        repos_iterator <- paste0(repos_table$organization, "/", repos_table$name)
+        repos_iterator <- paste0(repos_table$organization, "/", repos_table$repo_name)
         issues <- purrr::map_dfr(repos_iterator, function(repo_path) {
           issues_endpoint <- paste0(self$rest_api_url, "/repos/", repo_path, "/issues")
 
@@ -363,8 +354,8 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
     pull_commits_from_org = function(repos_table,
                                      date_from,
                                      date_until) {
-      repos_names <- repos_table$name
-      repo_fullnames <- paste0(repos_table$organization, "/", repos_table$name)
+      repos_names <- repos_table$repo_name
+      repo_fullnames <- paste0(repos_table$organization, "/", repos_table$repo_name)
 
       repos_list_with_commits <- purrr::map(repo_fullnames, function(repo_fullname) {
         commits_from_repo <- private$pull_commits_from_repo(

diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R
@@ -73,7 +73,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab",
         if (!private$scan_all) {
           cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{unique(repos_table$organization)}] Pulling contributors...")
         }
-        repo_iterator <- repos_table$id
+        repo_iterator <- repos_table$repo_id
         user_name <- rlang::expr(.$name)
         repos_table$contributors <- purrr::map_chr(repo_iterator, function(repos_id) {
           id <- gsub("gid://gitlab/Project/", "", repos_id)
@@ -194,7 +194,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab",
     # @return A list of repositories.
     search_repos_by_phrase = function(phrase,
                                       org,
-                                      language,
+                                      language = "All",
                                       page_max = 1e6) {
       page <- 1
       still_more_hits <- TRUE
@@ -223,7 +223,6 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab",
       repos_list <- resp_list %>%
         private$find_repos_by_id() %>%
         private$pull_repos_languages()
-
       return(repos_list)
     },
 
@@ -260,8 +259,8 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab",
     tailor_repos_info = function(projects_list) {
       projects_list <- purrr::map(projects_list, function(project) {
         list(
-          "id" = project$id,
-          "name" = project$name,
+          "repo_id" = project$id,
+          "repo_name" = project$name,
           "default_branch" = project$default_branch,
           "stars" = project$star_count,
           "forks" = project$fork_count,
@@ -282,7 +281,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab",
     # @return A table of repositories with added information on issues.
     pull_repos_issues = function(repos_table) {
       if (nrow(repos_table) > 0) {
-        issues <- purrr::map(repos_table$id, function(repos_id) {
+        issues <- purrr::map(repos_table$repo_id, function(repos_id) {
           id <- gsub("gid://gitlab/Project/", "", repos_id)
           issues_endpoint <- paste0(self$rest_api_url, "/projects/", id, "/issues_statistics")
 
@@ -325,8 +324,8 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab",
     pull_commits_from_org = function(repos_table,
                                      date_from,
                                      date_until) {
-      repos_names <- repos_table$name
-      projects_ids <- gsub("gid://gitlab/Project/", "", repos_table$id)
+      repos_names <- repos_table$repo_name
+      projects_ids <- gsub("gid://gitlab/Project/", "", repos_table$repo_id)
 
       repos_list_with_commits <- purrr::map(projects_ids, function(project_id) {
         commits_from_repo <- private$pull_commits_from_repo(

diff --git a/R/GQLQueryGitHub.R b/R/GQLQueryGitHub.R
@@ -170,6 +170,7 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub",
           repository(owner: $org, name: $repo) {
             id
             name
+            url
             object(expression: $file_path) {
               ... on Blob {
                 text
@@ -203,8 +204,8 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub",
           hasNextPage
         }
         nodes {
-          id
-          name
+          repo_id: id
+          repo_name: name
           default_branch: defaultBranchRef {
             name
           }

diff --git a/R/GQLQueryGitLab.R b/R/GQLQueryGitLab.R
@@ -42,8 +42,8 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab",
             }
             edges {
               node {
-                id
-                name
+                repo_id: id
+                repo_name: name
                 ... on Project {
                   repository {
                     rootRef
@@ -124,6 +124,7 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab",
                   node {
                     name
                     id
+                    webUrl
                     repository {
                       blobs(paths: $file_paths) {
                         nodes {
@@ -139,6 +140,28 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab",
             }
           }'
       )
+    },
+
+    #' @description Prepare query to get files in a standard filepath from
+    #'   GitLab repositories.
+    #' @return A query.
+    files_from_repo = function(){
+      'query GetFilesFromRepo($file_paths: [String!]!, $project_path: ID!) {
+              project(fullPath: $project_path) {
+                name
+                id
+                webUrl
+                repository {
+                  blobs(paths: $file_paths) {
+                    nodes {
+                      name
+                      rawBlob
+                      size
+                    }
+                  }
+                }
+              }
+          }'
     }
   )
 )
diff --git a/R/GitHost.R b/R/GitHost.R
@@ -385,23 +385,16 @@ GitHost <- R6::R6Class("GitHost",
 
     # @description Add `api_url` column to table.
     add_repo_api_url = function(repos_table){
-      if ("file_content" %in% colnames(repos_table)) {
-        repo_name <- rlang::expr("repository_name")
-        repo_id <- rlang::expr("repository_id")
-      } else {
-        repo_name <- rlang::expr("name")
-        repo_id <- rlang::expr("id")
-      }
       if (!is.null(repos_table) && nrow(repos_table) > 0) {
         repos_table <- if (private$host == "GitHub") {
           dplyr::mutate(
             repos_table,
-            api_url = paste0(private$api_url, "/repos/", organization, "/", eval(parse(text = repo_name))),
+            api_url = paste0(private$api_url, "/repos/", organization, "/", repo_name),
           )
         } else if (private$host == "GitLab") {
           dplyr::mutate(
             repos_table,
-            api_url = paste0(private$api_url, "/projects/", gsub("gid://gitlab/Project/", "", eval(parse(text = repo_id))))
+            api_url = paste0(private$api_url, "/projects/", stringr::str_match(repo_id, "[0-9].*"))
           )
         }
       }

diff --git a/R/GitStats.R b/R/GitStats.R
@@ -369,7 +369,7 @@ GitStats <- R6::R6Class("GitStats",
         if (!is.null(repos_using_package)) {
           repos_using_package$package_usage <- "library"
           repos_using_package <- repos_using_package %>%
-            dplyr::select(api_url, package_usage)
+            dplyr::select(repo_name, repo_url, api_url, package_usage)
         }
         return(repos_using_package)
       }) %>%
@@ -392,7 +392,7 @@ GitStats <- R6::R6Class("GitStats",
         repos_with_package$package_usage <- "import"
       }
       repos_with_package <- repos_with_package %>%
-        dplyr::select(api_url, package_usage)
+        dplyr::select(repo_name, repo_url, api_url, package_usage)
       return(repos_with_package)
     },
 

diff --git a/R/get_stats.R b/R/get_stats.R
@@ -8,7 +8,7 @@ get_repos_stats <- function(gitstats_obj){
   repos_data <- data.table::copy(get_repos(gitstats_obj))
   repos_stats <- repos_data %>%
     dplyr::mutate(
-      fullname = paste0(organization, "/", name)
+      fullname = paste0(organization, "/", repo_name)
     ) %>%
     dplyr::mutate(
       last_activity = difftime(