Major changes after review: improving handling of API requests (mainl…

…y errors), fixing search responses, changing logic of checking usage of a package (two separate functions); also as a side effect of previous attempts add possibility to pull files on given repos (although it is not used now, it may be in the future). Additionaly remove `silence` field from settings as there is no need to use it, change name of check_package to check_R_package.
r-world-devs · Dec 1, 2023 · 9faa730 · 9faa730
1 parent aae38f8
commit 9faa730
Show file tree

Hide file tree

Showing 23 changed files with 800 additions and 292 deletions.
diff --git a/NAMESPACE b/NAMESPACE
@@ -3,8 +3,9 @@
 S3method(gitstats_plot,commits_stats)
 S3method(gitstats_plot,repos_stats)
 export("%>%")
-export(check_package_usage)
+export(check_R_package_usage)
 export(create_gitstats)
+export(get_R_package_usage)
 export(get_commits)
 export(get_commits_stats)
 export(get_files)

diff --git a/NEWS.md b/NEWS.md
@@ -2,7 +2,7 @@
 
 ## Features:
 
-- added `check_package_usage()` function to pull repositories where code blobs with phrases related to using an R package are found (e.g. `library(package)`, `package::`) ([#326](https://github.com/r-world-devs/GitStats/issues/326)),
+- added `check_R_package_usage()` function to pull repositories where package name is found in DESCRIPTION or NAMESPACE files or code blobs with phrases related to using an R package (`library(package)`, `package::`) ([#326](https://github.com/r-world-devs/GitStats/issues/326)),
 - added `pull_files()` with `get_files()` to pull content of text files ([#200](https://github.com/r-world-devs/GitStats/issues/200)),
 - added a `default_branch` column to repositories output as a consequence of [#200](https://github.com/r-world-devs/GitStats/issues/200).
 

diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R
@@ -29,11 +29,12 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL",
      #' @param vars A list of named variables.
      #' @return A list.
      gql_response = function(gql_query, vars = "null") {
-       httr2::request(paste0(self$gql_api_url, "?")) %>%
-         httr2::req_headers("Authorization" = paste0("Bearer ", private$token)) %>%
-         httr2::req_body_json(list(query = gql_query, variables = vars)) %>%
-         httr2::req_perform() %>%
-         httr2::resp_body_json()
+       response <- private$perform_request(
+         gql_query = gql_query,
+         vars = vars
+       )
+       response_list <- httr2::resp_body_json(response)
+       return(response_list)
      },
 
      #' @description Get information on users in the form of table
@@ -51,14 +52,17 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL",
      #'   an organization in a table format.
      #' @param org An organization.
      #' @param file_path A file path.
+     #' @param pulled_repos Optional parameter to pass repository output object.
+     #' @param settings A list of  `GitStats` settings.
      #' @return A table.
-     pull_files = function(org, file_path) {
+     pull_files = function(org, file_path, pulled_repos = NULL) {
        if (!private$scan_all) {
          cli::cli_alert_info("[Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling {file_path} files...")
        }
        files_table <- private$pull_file_from_org(
          org = org,
-         file_path = file_path
+         file_path = file_path,
+         pulled_repos = pulled_repos
        ) %>%
          private$prepare_files_table(
            org = org,
@@ -75,6 +79,18 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL",
      # @field A boolean.
      scan_all = FALSE,
 
+     perform_request = function(gql_query, vars) {
+       response <- httr2::request(paste0(self$gql_api_url, "?")) %>%
+         httr2::req_headers("Authorization" = paste0("Bearer ", private$token)) %>%
+         httr2::req_body_json(list(query = gql_query, variables = vars)) %>%
+         httr2::req_retry(
+           is_transient = ~ httr2::resp_status(.x) == "400|502",
+           max_seconds = 60
+         ) %>%
+         httr2::req_perform()
+       return(response)
+     },
+
      # @description A method to pull information on user.
      # @param username A login.
      # @return A user response.

diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R
@@ -54,7 +54,7 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
                           settings) {
       if (settings$search_param %in% c("org", "team")) {
         if (settings$search_param == "org") {
-          if (!private$scan_all && !settings$silence) {
+          if (!private$scan_all) {
             cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...")
           }
           repos_table <- private$pull_repos_from_org(
@@ -63,7 +63,7 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
           ) %>%
             private$prepare_repos_table()
         } else {
-          if (!private$scan_all && !settings$silence) {
+          if (!private$scan_all) {
             cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling repositories...")
           }
           repos_table <- private$pull_repos_from_team(
@@ -106,13 +106,12 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
                             settings) {
       repos_table <- self$pull_repos(
         org = org,
-        settings = list(search_param = "org",
-                        silence = TRUE)
+        settings = list(search_param = "org")
       )
       repos_names <- repos_table$name
 
       if (settings$search_param == "org") {
-        if (!private$scan_all  && !settings$silence) {
+        if (!private$scan_all) {
           cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling commits...")
         }
         repos_list_with_commits <- private$pull_commits_from_repos(
@@ -123,7 +122,7 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
         )
       }
       if (settings$search_param == "team") {
-        if (!private$scan_all && !settings$silence) {
+        if (!private$scan_all) {
           cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling commits...")
         }
         repos_list_with_commits <- private$pull_commits_from_repos(
@@ -459,31 +458,43 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
       return(user_table)
     },
 
-    # @description Pull all given files from all repositories of an organization.
+    # @description Pull all given files from all repositories of an
+    #   organization.
     # @param org An organization.
     # @param file_path Path to a file.
+    # @param pulled_repos Optional, if not empty, function will make use of the
+    #   argument to iterate over it when pulling files.
     # @return A response in a list form.
-    pull_file_from_org = function(org, file_path) {
-      repos_list <- private$pull_repos_from_org(
-        from = "org",
-        org = org
-      )
-      repositories <- purrr::map(repos_list, ~ .$name)
-      def_branches <- purrr::map(repos_list, ~ .$default_branch$name)
-      files_list <- purrr::map2(repositories, def_branches, function(repository, def_branch) {
-        files_query <- self$gql_query$files_by_repo()
-        files_response <- self$gql_response(
-          gql_query = files_query,
-          vars = list(
-            "org" = org,
-            "repo" = repository,
-            "file_path" = paste0(def_branch, ":", file_path)
-          )
+    pull_file_from_org = function(org, file_path, pulled_repos = NULL) {
+      if (is.null(pulled_repos)) {
+        repos_list <- private$pull_repos_from_org(
+          from = "org",
+          org = org
         )
-      }) %>%
-        purrr::map(~ .$data$repository)
-      names(files_list) <- repositories
-      files_list <- purrr::discard(files_list, ~ length(.$object) == 0)
+        repositories <- purrr::map(repos_list, ~ .$name)
+        def_branches <- purrr::map(repos_list, ~ .$default_branch$name)
+      } else {
+        repositories <- pulled_repos$name
+        def_branches <- pulled_repos$default_branch
+      }
+      files_list <- purrr::map(file_path, function(file_path) {
+        files_list <- purrr::map2(repositories, def_branches, function(repository, def_branch) {
+          files_query <- self$gql_query$files_by_repo()
+          files_response <- self$gql_response(
+            gql_query = files_query,
+            vars = list(
+              "org" = org,
+              "repo" = repository,
+              "file_path" = paste0(def_branch, ":", file_path)
+            )
+          )
+        }) %>%
+          purrr::map(~ .$data$repository)
+        names(files_list) <- repositories
+        files_list <- purrr::discard(files_list, ~ length(.$object) == 0)
+        return(files_list)
+      })
+      names(files_list) <- file_path
       return(files_list)
     },
 
@@ -493,16 +504,19 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
     # @return A table with information on files.
     prepare_files_table = function(files_response, org, file_path) {
       if (!is.null(files_response)) {
-        files_table <- purrr::imap(files_response, function(repository, name) {
-          data.frame(
-            "repository_name" = repository$name,
-            "repository_id" = repository$id,
-            "organization" = org,
-            "file_path" = file_path,
-            "file_content" = repository$object$text,
-            "file_size" = repository$object$byteSize,
-            "api_url" = self$gql_api_url
-          )
+        files_table <- purrr::map(file_path, function(file) {
+          purrr::imap(files_response[[file]], function(repository, name) {
+            data.frame(
+              "repository_name" = repository$name,
+              "repository_id" = repository$id,
+              "organization" = org,
+              "file_path" = file,
+              "file_content" = repository$object$text,
+              "file_size" = repository$object$byteSize,
+              "api_url" = self$gql_api_url
+            )
+          }) %>%
+            purrr::list_rbind()
         }) %>%
           purrr::list_rbind()
       } else {

diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R
@@ -54,10 +54,10 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab",
      #' @param settings A list of  `GitStats` settings.
      #' @return A table.
      pull_repos = function(org,
-                          settings) {
+                           settings) {
        org <- gsub("%2f", "/", org)
        if (settings$search_param == "org") {
-         if (!private$scan_all && !settings$silence) {
+         if (!private$scan_all) {
            cli::cli_alert_info("[GitLab][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...")
          }
          repos_table <- private$pull_repos_from_org(
@@ -226,45 +226,75 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab",
      # @description Pull all given files from all repositories of a group.
      # @param org An organization.
      # @param file_path Path to a file.
+     # @param pulled_repos Optional, if not empty, function will make use of the
+     #   argument to iterate over it when pulling files.
      # @return A response in a list form.
-     pull_file_from_org = function(org, file_path) {
-       full_files_list <- list()
-       next_page <- TRUE
-       end_cursor <- ""
-       while (next_page) {
-         files_query <- self$gql_query$files_by_org(
-           end_cursor = end_cursor
-         )
+     pull_file_from_org = function(org, file_path, pulled_repos = NULL) {
+       if (!is.null(pulled_repos)) {
+        full_files_list <- private$pull_file_from_repos(
+          file_path = file_path,
+          repos_table = pulled_repos
+        )
+       } else {
+         full_files_list <- list()
+         next_page <- TRUE
+         end_cursor <- ""
+         while (next_page) {
+           files_query <- self$gql_query$files_by_org(
+             end_cursor = end_cursor
+           )
+           files_response <- self$gql_response(
+             gql_query = files_query,
+             vars = list(
+               "org" = org,
+               "file_paths" = file_path
+             )
+           )
+           if (length(files_response$data$group) == 0) {
+             cli::cli_alert_danger("Empty")
+           }
+           projects <- files_response$data$group$projects
+           files_list <- purrr::map(projects$edges, function(edge) {
+             edge$node
+           }) %>%
+             purrr::discard(~ length(.$repository$blobs$nodes) == 0)
+           if (is.null(files_list)) files_list <- list()
+           if (length(files_list) > 0) {
+             next_page <- files_response$pageInfo$hasNextPage
+           } else {
+             next_page <- FALSE
+           }
+           if (is.null(next_page)) next_page <- FALSE
+           if (next_page) {
+             end_cursor <- files_response$pageInfo$endCursor
+           } else {
+             end_cursor <- ""
+           }
+           full_files_list <- append(full_files_list, files_list)
+         }
+       }
+       return(full_files_list)
+     },
+
+     # @description Pull all given files from given repositories.
+     # @param file_path Path to a file.
+     # @param repos_table Repositories table.
+     # @return A response in a list form.
+     pull_file_from_repos = function(file_path, repos_table) {
+       files_list <- purrr::map(repos_table$repo_url, function(repo_url) {
+         files_query <- self$gql_query$files_from_repo()
          files_response <- self$gql_response(
            gql_query = files_query,
            vars = list(
-             "org" = org,
-             "file_paths" = file_path
+             "file_paths" = file_path,
+             "project_path" = stringr::str_replace(repo_url, ".*(?<=.com/)", "")
            )
          )
-         if (length(files_response$data$group) == 0) {
-           cli::cli_abort("Empty")
-         }
-         projects <- files_response$data$group$projects
-         files_list <- purrr::map(projects$edges, function(edge) {
-           edge$node
-         }) %>%
-           purrr::discard(~ length(.$repository$blobs$nodes) == 0)
-         if (is.null(files_list)) files_list <- list()
-         if (length(files_list) > 0) {
-           next_page <- files_response$pageInfo$hasNextPage
-         } else {
-           next_page <- FALSE
-         }
-         if (is.null(next_page)) next_page <- FALSE
-         if (next_page) {
-           end_cursor <- files_response$pageInfo$endCursor
-         } else {
-           end_cursor <- ""
-         }
-         full_files_list <- append(full_files_list, files_list)
-       }
-       return(full_files_list)
+         return(files_response)
+       }) %>%
+         purrr::discard(~ length(.$data$project$repository$blobs$nodes) == 0) %>%
+         purrr::map(~ .$data$project)
+       return(files_list)
      },
 
      # @description Prepare files table.