From bbe874305458bb43b886b92cb14709bb6b73d9ca Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 1 Oct 2024 08:12:18 +0000 Subject: [PATCH 01/94] Mock commit responses. --- R/GitHostGitLab.R | 4 +- tests/testthat/helper-fixtures.R | 61 ++++++++++++++++++++++++ tests/testthat/test-get_commits-GitHub.R | 18 +------ tests/testthat/test-get_commits-GitLab.R | 22 +-------- 4 files changed, 66 insertions(+), 39 deletions(-) diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index 5e444564..bf1ffe45 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -355,8 +355,8 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", # A helper to turn list of data.frames into one data.frame prepare_commits_table = function(commits_list) { - commits_dt <- purrr::map(commits_list, function(x) { - purrr::map(x, ~ data.frame(.)) %>% + commits_dt <- purrr::map(commits_list, function(commit) { + purrr::map(commit, ~ data.frame(.)) %>% purrr::list_rbind() }) %>% purrr::list_rbind() diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 13298240..3026d0d2 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -30,6 +30,67 @@ test_fixtures$half_empty_gql_response <- list( ) ) +commit_edge <- list( + "node" = list( + "id" = "xxx", + "committed_date" = "2023-01-25T10:26:41Z", + "author" = list( + "name" = "Maciej Banas", + "user" = list( + "name" = "Maciej Banas", + "login" = "maciekbanas" + ) + ), + "additions" = 5L, + "deletions" = 8L + ) +) + +test_fixtures$github_commits_response <- list( + "data" = list( + "repository" = list( + "defaultBranchRef" = list( + "target" = list( + "history" = list( + "edges" = list( + rep(commit_edge, 5) + ) + ) + ) + ) + ) + ) +) + +gitlab_commit <- list( + "id" = "xxxxxxxxxxxxxxxxx", + "short_id" = "xxx", + "created_at" = "2023-04-05T12:07:50.000+00:00", + "parent_ids" = list( + "iiiiiiiiiiiiiii" + ), + "title" = "Test title", + "message" = "Test title", + "author_name" = "Maciej Banas", + "author_email" = "testmail@test.com", + "authored_date" = "2023-04-05T12:07:50.000+00:00", + "committer_name" = "Maciej Banas", + "committer_email" = "testmail@test.com", + "committed_date" = "2023-04-05T12:07:50.000+00:00", + "trailers" = list(), + "extedned_trailers" = list(), + "web_url" = "https://test_url.com", + "stats" = list( + "additions" = 1L, + "deletions" = 0L, + "total" = 1L + ) +) + +test_fixtures$gitlab_commits_response <- list( + rep(gitlab_commit, 5) +) + test_fixtures$github_file_response <- list( "data" = list( "repository" = list( diff --git a/tests/testthat/test-get_commits-GitHub.R b/tests/testthat/test-get_commits-GitHub.R index 5c239cb5..88f7851f 100644 --- a/tests/testthat/test-get_commits-GitHub.R +++ b/tests/testthat/test-get_commits-GitHub.R @@ -12,21 +12,11 @@ test_that("commits_by_repo GitHub query is built properly", { test_mocker$cache(gh_commits_by_repo_query) }) -test_that("GitHub GraphQL API returns commits response", { - gh_commits_by_repo_gql_response <- test_graphql_github$gql_response( - test_mocker$use("gh_commits_by_repo_query") - ) - expect_gh_commit_gql_response( - gh_commits_by_repo_gql_response$data$repository$defaultBranchRef$target$history$edges[[1]] - ) - test_mocker$cache(gh_commits_by_repo_gql_response) -}) - test_that("`get_commits_page_from_repo()` pulls commits page from repository", { mockery::stub( test_graphql_github_priv$get_commits_page_from_repo, "self$gql_response", - test_mocker$use("gh_commits_by_repo_gql_response") + test_fixtures$github_commits_response ) commits_page <- test_graphql_github_priv$get_commits_page_from_repo( org = "r-world-devs", @@ -41,14 +31,10 @@ test_that("`get_commits_page_from_repo()` pulls commits page from repository", { }) test_that("`get_commits_from_one_repo()` prepares formatted list", { - # overcome of infinite loop in get_commits_from_repo - commits_page <- test_mocker$use("commits_page") - commits_page$data$repository$defaultBranchRef$target$history$pageInfo$hasNextPage <- FALSE - mockery::stub( test_graphql_github_priv$get_commits_from_one_repo, "private$get_commits_page_from_repo", - commits_page + test_mocker$use("commits_page") ) commits_from_repo <- test_graphql_github_priv$get_commits_from_one_repo( org = "r-world-devs", diff --git a/tests/testthat/test-get_commits-GitLab.R b/tests/testthat/test-get_commits-GitLab.R index 32b9fa6f..7a55e381 100644 --- a/tests/testthat/test-get_commits-GitLab.R +++ b/tests/testthat/test-get_commits-GitLab.R @@ -1,28 +1,8 @@ -test_that("GitLab REST API returns commits response", { - gl_commits_rest_response_repo_1 <- test_rest_gitlab$response( - "https://gitlab.com/api/v4/projects/44293594/repository/commits?since='2023-01-01T00:00:00'&until='2023-04-20T00:00:00'&with_stats=true" - ) - expect_gl_commit_rest_response( - gl_commits_rest_response_repo_1 - ) - test_mocker$cache(gl_commits_rest_response_repo_1) - - gl_commits_rest_response_repo_2 <- test_rest_gitlab$response( - "https://gitlab.com/api/v4/projects/44346961/repository/commits?since='2023-01-01T00:00:00'&until='2023-04-20T00:00:00'&with_stats=true" - ) - expect_gl_commit_rest_response( - gl_commits_rest_response_repo_2 - ) - test_mocker$cache(gl_commits_rest_response_repo_2) -}) - test_that("`get_commits_from_repos()` pulls commits from repo", { - gl_commits_repo_1 <- test_mocker$use("gl_commits_rest_response_repo_1") - mockery::stub( test_rest_gitlab$get_commits_from_repos, "private$get_commits_from_one_repo", - gl_commits_repo_1 + test_fixtures$gitlab_commits_response ) repos_names <- c("mbtests%2Fgitstatstesting", "mbtests%2Fgitstats-testing-2") gl_commits_org <- test_rest_gitlab$get_commits_from_repos( From 9d22d4858f7783121fd37b754532e503fe976500 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 2 Oct 2024 14:11:45 +0000 Subject: [PATCH 02/94] Mock tests for GitHub releases. --- tests/testthat/helper-fixtures.R | 41 ++++++++++++++++++++++++ tests/testthat/test-get_release-GitHub.R | 9 ++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 3026d0d2..e51b9467 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -356,3 +356,44 @@ test_fixtures$github_files_tree_response <- list( ) ) ) + +test_fixtures$github_releases_from_org <- list( + "data" = list( + "repository" = list( + "name" = "TestProject1", + "url" = "test_url", + "releases" = list( + "nodes" = list( + list( + "name" = "2.1.0", + "tagName" = "v2.1.0", + "publishedAt" = "2024-01-01T00:00:00Z", + "url" = "https://test_url/tag/v2.1.0", + "description" = "Great features come with this release." + ), + list( + "name" = "1.1.1", + "tagName" = "v1.1.1", + "publishedAt" = "2023-09-28T00:00:00Z", + "url" = "https://test_url/tag/v1.1.0", + "description" = "Great features come with this release." + ), + list( + "name" = "0.1.0", + "tagName" = "v0.1.0", + "publishedAt" = "2023-04-01T00:00:00Z", + "url" = "https://test_url/tag/v0.1.0", + "description" = "Great features come with this release." + ), + list( + "name" = "0.1.1", + "tagName" = "v0.1.1", + "publishedAt" = "2023-05-02T00:00:00Z", + "url" = "https://test_url/tag/v0.1.0", + "description" = "Great features come with this release." + ) + ) + ) + ) + ) +) diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index 72593599..088ff0ea 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -7,9 +7,14 @@ test_that("releases query is built properly", { }) test_that("`get_releases_from_org()` pulls releases from the repositories", { + mockery::stub( + test_graphql_github$get_release_logs_from_org, + "self$gql_response", + test_fixtures$github_releases_from_org + ) releases_from_repos <- test_graphql_github$get_release_logs_from_org( - repos_names = c("GitStats", "shinyCohortBuilder"), - org = "r-world-devs" + repos_names = c("TestProject1", "TestProject2"), + org = "test_org" ) expect_github_releases_response(releases_from_repos) test_mocker$cache(releases_from_repos) From 8ad995c7cb4a97617c0340c45190a67482771ebd Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 2 Oct 2024 14:49:04 +0000 Subject: [PATCH 03/94] Mock tests for GitLab releases. --- tests/testthat/helper-expect-responses.R | 20 ++++++++++ tests/testthat/helper-fixtures.R | 51 +++++++++++++++++++++++- tests/testthat/test-get_release-GitHub.R | 2 +- tests/testthat/test-get_release-GitLab.R | 46 +++++++++++++++++++++ 4 files changed, 117 insertions(+), 2 deletions(-) diff --git a/tests/testthat/helper-expect-responses.R b/tests/testthat/helper-expect-responses.R index 883e4269..1291fecc 100644 --- a/tests/testthat/helper-expect-responses.R +++ b/tests/testthat/helper-expect-responses.R @@ -242,3 +242,23 @@ expect_github_releases_response <- function(object) { }) }) } + +expect_gitlab_releases_response <- function(object) { + expect_type( + object, + "list" + ) + purrr::walk(object, function(response) { + expect_gt(length(response), 0) + expect_list_contains( + response$data$project, + c("releases") + ) + purrr::walk(response$data$project$releases$nodes, function(node) { + expect_list_contains( + node, + c("name", "tagName", "releasedAt", "links", "description") + ) + }) + }) +} diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index e51b9467..fe98c832 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -357,7 +357,7 @@ test_fixtures$github_files_tree_response <- list( ) ) -test_fixtures$github_releases_from_org <- list( +test_fixtures$github_releases_from_repo <- list( "data" = list( "repository" = list( "name" = "TestProject1", @@ -397,3 +397,52 @@ test_fixtures$github_releases_from_org <- list( ) ) ) + +test_fixtures$gitlab_releases_from_repo <- list( + "data" = list( + "project" = list( + "name" = "TestProject", + "webUrl" = "test_url", + "releases" = list( + "nodes" = list( + list( + "name" = "TestProject 1.0.0", + "tagName" = "1.0.0", + "releasedAt" = "2024-06-08T00:00:00+02:00", + "links" = list( + "selfUrl" = "test_url/releases/1.0.0" + ), + "description" = "This release comes with awesome features." + ), + list( + "name" = "TestProject 2.0.0", + "tagName" = "2.0.0", + "releasedAt" = "2024-08-08T00:00:00+02:00", + "links" = list( + "selfUrl" = "test_url/releases/2.0.0" + ), + "description" = "This release comes with awesome features." + ), + list( + "name" = "TestProject 0.1.0", + "tagName" = "0.1.0", + "releasedAt" = "2023-06-08T00:00:00+02:00", + "links" = list( + "selfUrl" = "test_url/releases/0.1.0" + ), + "description" = "This release comes with awesome features." + ), + list( + "name" = "TestProject 3.0.0", + "tagName" = "3.0.0", + "releasedAt" = "2023-09-08T00:00:00+02:00", + "links" = list( + "selfUrl" = "test_url/releases/3.0.0" + ), + "description" = "This release comes with awesome features." + ) + ) + ) + ) + ) +) diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index 088ff0ea..5f25aff3 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -10,7 +10,7 @@ test_that("`get_releases_from_org()` pulls releases from the repositories", { mockery::stub( test_graphql_github$get_release_logs_from_org, "self$gql_response", - test_fixtures$github_releases_from_org + test_fixtures$github_releases_from_repo ) releases_from_repos <- test_graphql_github$get_release_logs_from_org( repos_names = c("TestProject1", "TestProject2"), diff --git a/tests/testthat/test-get_release-GitLab.R b/tests/testthat/test-get_release-GitLab.R index ad1d3918..15b18ca1 100644 --- a/tests/testthat/test-get_release-GitLab.R +++ b/tests/testthat/test-get_release-GitLab.R @@ -5,3 +5,49 @@ test_that("releases query is built properly", { gl_releases_query ) }) + +test_that("`get_releases_from_org()` pulls releases from the repositories", { + mockery::stub( + test_graphql_gitlab$get_release_logs_from_org, + "self$gql_response", + test_fixtures$gitlab_releases_from_repo + ) + releases_from_repos <- test_graphql_gitlab$get_release_logs_from_org( + repos_names = c("TestProject1", "TestProject2"), + org = "test_org" + ) + expect_gitlab_releases_response(releases_from_repos) + test_mocker$cache(releases_from_repos) +}) + + +test_that("`prepare_releases_table()` prepares releases table", { + releases_table <- gitlab_testhost_priv$prepare_releases_table( + releases_response = test_mocker$use("releases_from_repos"), + org = "r-world-devs", + date_from = "2023-08-01", + date_until = "2024-06-30" + ) + expect_releases_table(releases_table) + expect_gt(min(releases_table$published_at), as.POSIXct("2023-08-01")) + expect_lt(max(releases_table$published_at), as.POSIXct("2024-06-30")) + test_mocker$cache(releases_table) +}) + +test_that("`get_release_logs()` pulls release logs in the table format", { + mockery::stub( + gitlab_testhost$get_release_logs, + "private$prepare_releases_table", + test_mocker$use("releases_table") + ) + releases_table <- gitlab_testhost$get_release_logs( + since = "2023-08-01", + until = "2024-06-30", + verbose = FALSE, + progress = FALSE + ) + expect_releases_table(releases_table) + expect_gt(min(releases_table$published_at), as.POSIXct("2023-08-01")) + expect_lt(max(releases_table$published_at), as.POSIXct("2024-06-30")) + test_mocker$cache(releases_table) +}) From 5588b8b603f6b7fd696d8848894556a3680babab Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 3 Oct 2024 09:44:59 +0000 Subject: [PATCH 04/94] Mock repos responses for tests. --- R/GitHostGitHub.R | 1 + tests/testthat/helper-fixtures.R | 113 ++++++++++++++++++++++++- tests/testthat/test-get_repos-GitHub.R | 20 +---- tests/testthat/test-get_repos-GitLab.R | 21 ++--- 4 files changed, 120 insertions(+), 35 deletions(-) diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index b2df0367..37942bee 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -161,6 +161,7 @@ GitHostGitHub <- R6::R6Class( default_branch, .after = repo_name ) + return(repo) }) } else { repos_table <- NULL diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index fe98c832..4b61bef4 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -30,7 +30,116 @@ test_fixtures$half_empty_gql_response <- list( ) ) -commit_edge <- list( +github_repository_node <- list( + "repo_id" = "xyz", + "repo_name" = "TestRepo", + "default_branch" = list( + "name" = "main" + ), + "stars" = 10, + "forks" = 2, + "created_at" = "2022-04-20T00:00:00Z", + "last_activity_at" = "2023-04-20T00:00:00Z", + "languages" = list( + "nodes" = list( + list( + "name" = "R" + ), + list( + "name" = "CSS" + ), + list( + "name" = "JavaScript" + ) + ) + ), + "issues_open" = list( + "totalCount" = 10 + ), + "issues_closed" = list( + "totalCount" = 5 + ), + "organization" = list( + "login" = "test_org" + ), + "repo_url" = "https://test_url" +) + +test_fixtures$github_repos_by_org_response <- list( + "data" = list( + "repositoryOwner" = list( + "repositories" = list( + "totalCount" = 5, + "pageInfo" = list( + "endCursor" = "xyx", + "hasNextPage" = FALSE + ), + "nodes" = list( + github_repository_node, + github_repository_node, + github_repository_node, + github_repository_node, + github_repository_node + ) + ) + ) + ) +) + +gitlab_project_node <- list( + "node" = list( + "repo_id" = "gid://gitlab/Project/61399846", + "repo_name" = "test_repo", + "repo_path" = "test_repo", + "repository" = list( + "rootRef" = "main" + ), + "stars" = 8, + "forks" = 3, + "created_at" = "2023-09-18T00:00:00Z", + "last_activity_at" = "2024-09-18T00:00:00Z", + "languages" = list( + list( + "name" = "Python" + ), + list( + "name" = "R" + ) + ), + "issues" = list( + "all" = 10, + "closed" = 8, + "opened" = 2 + ), + "group" = list( + "path" = "test_group" + ), + "repo_url" = "https://test_gitlab_url.com" + ) +) + +test_fixtures$gitlab_repos_by_org_response <- list( + "data" = list( + "group" = list( + "projects" = list( + "count" = 5, + "pageInfo" = list( + "hasNextPage" = FALSE, + "endCursor" = "xyz" + ), + "edges" = list( + gitlab_project_node, + gitlab_project_node, + gitlab_project_node, + gitlab_project_node, + gitlab_project_node + ) + ) + ) + ) +) + +github_commit_edge <- list( "node" = list( "id" = "xxx", "committed_date" = "2023-01-25T10:26:41Z", @@ -53,7 +162,7 @@ test_fixtures$github_commits_response <- list( "target" = list( "history" = list( "edges" = list( - rep(commit_edge, 5) + rep(github_commit_edge, 5) ) ) ) diff --git a/tests/testthat/test-get_repos-GitHub.R b/tests/testthat/test-get_repos-GitHub.R index b8923055..5e641963 100644 --- a/tests/testthat/test-get_repos-GitHub.R +++ b/tests/testthat/test-get_repos-GitHub.R @@ -7,25 +7,14 @@ test_that("repos_by_org query is built properly", { test_mocker$cache(gh_repos_by_org_query) }) -test_that("GitHub repos response to query works", { - gh_repos_by_org_gql_response <- test_graphql_github$gql_response( - test_mocker$use("gh_repos_by_org_query"), - vars = list(org = "r-world-devs") - ) - expect_gh_repos_gql_response( - gh_repos_by_org_gql_response - ) - test_mocker$cache(gh_repos_by_org_gql_response) -}) - test_that("`get_repos_page()` pulls repos page from GitHub organization", { mockery::stub( test_graphql_github_priv$get_repos_page, "self$gql_response", - test_mocker$use("gh_repos_by_org_gql_response") + test_fixtures$github_repos_by_org_response ) gh_repos_page <- test_graphql_github_priv$get_repos_page( - org = "r-world-devs" + org = "test_org" ) expect_gh_repos_gql_response( gh_repos_page @@ -40,7 +29,7 @@ test_that("`get_repos_from_org()` prepares formatted list", { test_mocker$use("gh_repos_page") ) gh_repos_from_org <- test_graphql_github$get_repos_from_org( - org = "r-world-devs" + org = "test_org" ) expect_list_contains( gh_repos_from_org[[1]], @@ -56,9 +45,6 @@ test_that("`get_repos_from_org()` prepares formatted list", { # REST Engine search repos by code test_that("`response()` returns search response from GitHub's REST API", { - # search_endpoint <- "https://api.github.com/search/code?q=shiny+user:r-world-devs" - # test_mocker$cache(search_endpoint) - # gh_search_response_raw <- test_rest_github$response(search_endpoint) gh_search_response_raw <- test_fixtures$github_search_response expect_gh_search_response(gh_search_response_raw[["items"]]) test_mocker$cache(gh_search_response_raw) diff --git a/tests/testthat/test-get_repos-GitLab.R b/tests/testthat/test-get_repos-GitLab.R index 5105cc40..2d7438a4 100644 --- a/tests/testthat/test-get_repos-GitLab.R +++ b/tests/testthat/test-get_repos-GitLab.R @@ -7,25 +7,14 @@ test_that("repos queries are built properly", { test_mocker$cache(gl_repos_by_org_query) }) -test_that("GitLab repos response to query works", { - gl_repos_by_org_gql_response <- test_graphql_gitlab$gql_response( - gql_query = test_mocker$use("gl_repos_by_org_query"), - vars = list(org = "mbtests") - ) - expect_gl_repos_gql_response( - gl_repos_by_org_gql_response - ) - test_mocker$cache(gl_repos_by_org_gql_response) -}) - test_that("`get_repos_page()` pulls repos page from GitLab group", { mockery::stub( test_graphql_gitlab_priv$get_repos_page, "self$gql_response", - test_mocker$use("gl_repos_by_org_gql_response") + test_fixtures$gitlab_repos_by_org_response ) gl_repos_page <- test_graphql_gitlab_priv$get_repos_page( - org = "mbtests" + org = "test_org" ) expect_gl_repos_gql_response( gl_repos_page @@ -40,7 +29,7 @@ test_that("`get_repos_from_org()` prepares formatted list", { test_mocker$use("gl_repos_page") ) gl_repos_from_org <- test_graphql_gitlab$get_repos_from_org( - org = "mbtests" + org = "test_org" ) expect_equal( names(gl_repos_from_org[[1]]$node), @@ -60,7 +49,7 @@ test_that("`get_repos_from_org()` does not fail when GraphQL response is not com test_fixtures$empty_gql_response ) gl_repos_from_org <- test_graphql_gitlab$get_repos_from_org( - org = "mbtests" + org = "test_org" ) expect_type( gl_repos_from_org, @@ -76,7 +65,7 @@ test_that("`get_repos_from_org()` does not fail when GraphQL response is not com test_fixtures$half_empty_gql_response ) gl_repos_from_org <- test_graphql_gitlab$get_repos_from_org( - org = "mbtests" + org = "test_org" ) expect_type( gl_repos_from_org, From 0a89ababd9c237452a8802ea1781fa8c890cad8f Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 4 Oct 2024 07:58:25 +0000 Subject: [PATCH 05/94] Mock repos urls, users and GitLab commits method for geting authors of commits. --- R/EngineRestGitLab.R | 86 +++++++++++---------- tests/testthat/helper-fixtures.R | 81 +++++++++++++++++++ tests/testthat/test-get_commits-GitLab.R | 53 +++++++++---- tests/testthat/test-get_urls_repos-GitHub.R | 10 +++ tests/testthat/test-get_urls_repos-GitLab.R | 38 ++++++++- tests/testthat/test-get_user-GitHub.R | 22 ++---- tests/testthat/test-get_user-GitLab.R | 32 ++------ 7 files changed, 224 insertions(+), 98 deletions(-) diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 2018b670..00b93618 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -158,46 +158,10 @@ EngineRestGitLab <- R6::R6Class( if (verbose) { cli::cli_alert_info("Looking up for authors' names and logins...") } - authors_dict <- purrr::map(unique(commits_table$author), function(author) { - author <- url_encode(author) - search_endpoint <- paste0( - self$rest_api_url, - "/search?scope=users&search=%22", author, "%22" - ) - user_response <- list() - try({ - user_response <- self$response(endpoint = search_endpoint) - }, silent = TRUE) - if (length(user_response) == 0) { - author <- stringi::stri_trans_general(author, "Latin-ASCII") - search_endpoint <- paste0( - self$rest_api_url, - "/search?scope=users&search=%22", author, "%22" - ) - try({ - user_response <- self$response(endpoint = search_endpoint) - }, silent = TRUE) - } - if (!is.null(user_response) && length(user_response) > 1) { - user_response <- purrr::keep(user_response, ~ grepl(author, .$name)) - } - if (is.null(user_response) || length(user_response) == 0) { - user_tbl <- tibble::tibble( - author = URLdecode(author), - author_login = NA, - author_name = NA - ) - } else { - user_tbl <- tibble::tibble( - author = URLdecode(author), - author_login = user_response[[1]]$username, - author_name = user_response[[1]]$name - ) - } - return(user_tbl) - }, .progress = progress) %>% - purrr::list_rbind() - + authors_dict <- private$get_authors_dict( + commits_table = commits_table, + progress = progress + ) commits_table <- commits_table %>% dplyr::mutate( author_login = NA, @@ -423,6 +387,48 @@ EngineRestGitLab <- R6::R6Class( FALSE }) |> purrr::discard(is.null) + }, + + get_authors_dict = function(commits_table, progress) { + purrr::map(unique(commits_table$author), function(author) { + author <- url_encode(author) + search_endpoint <- paste0( + self$rest_api_url, + "/search?scope=users&search=%22", author, "%22" + ) + user_response <- list() + try({ + user_response <- self$response(endpoint = search_endpoint) + }, silent = TRUE) + if (length(user_response) == 0) { + author <- stringi::stri_trans_general(author, "Latin-ASCII") + search_endpoint <- paste0( + self$rest_api_url, + "/search?scope=users&search=%22", author, "%22" + ) + try({ + user_response <- self$response(endpoint = search_endpoint) + }, silent = TRUE) + } + if (!is.null(user_response) && length(user_response) > 1) { + user_response <- purrr::keep(user_response, ~ grepl(author, .$name)) + } + if (is.null(user_response) || length(user_response) == 0) { + user_tbl <- tibble::tibble( + author = URLdecode(author), + author_login = NA, + author_name = NA + ) + } else { + user_tbl <- tibble::tibble( + author = URLdecode(author), + author_login = user_response[[1]]$username, + author_name = user_response[[1]]$name + ) + } + return(user_tbl) + }, .progress = progress) %>% + purrr::list_rbind() } ) ) diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 4b61bef4..8be5d657 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -30,6 +30,29 @@ test_fixtures$half_empty_gql_response <- list( ) ) +test_fixtures$github_repositories_rest_response <- list( + list( + "html_url" = "https://github.com/r-world-devs/GitStats", + "url" = "https://api.github.com/repos/r-world-devs/GitStats" + ), + list( + "html_url" = "https://github.com/openpharma/DataFakeR", + "url" = "https://api.github.com/repos/openpharma/DataFakeR" + ) +) + + +test_fixtures$gitlab_repositories_rest_response <- list( + list( + "_links" = list("self" = "https://gitlab.com/api/v4/projects/43400864"), + "web_url" = "https://gitlab.com/mbtests/gitstats-testing-2" + ), + list( + "_links" = list("self" = "https://gitlab.com/api/v4/projects/43398933"), + "web_url" = "https://gitlab.com/mbtests/gitstatstesting" + ) +) + github_repository_node <- list( "repo_id" = "xyz", "repo_name" = "TestRepo", @@ -555,3 +578,61 @@ test_fixtures$gitlab_releases_from_repo <- list( ) ) ) + +test_fixtures$github_user_response <- list( + "data" = list( + "user" = list( + "id" = "xxx", + "name" = "Mr Test", + "login" = "testlogin", + "email" = "", + "location" = NULL, + "starred_repos" = list( + "totalCount" = 15L + ), + "contributions" = list( + "totalCommitContributions" = 100L, + "totalIssueContributions" = 50L, + "totalPullRequestContributions" = 25L, + "totalPullRequestReviewContributions" = 5L + ), + "avatar_url" = "", + "web_url" = NULL + ) + ) +) + +test_fixtures$gitlab_user_response <- list( + "data" = list( + "user" = list( + "id" = "xxx", + "name" = "Mr Test", + "login" = "testlogin", + "email" = "", + "location" = NULL, + "starred_repos" = list( + "count" = 15L + ), + "pull_requests" = list( + "count" = 2L + ), + "reviews" = list( + "count" = 5L + ), + "avatar_url" = "", + "web_url" = "https://gitlab.com/maciekbanas" + ) + ) +) + +test_fixtures$gitlab_user_search_response <- list( + list( + "id" = "xxx", + "username" = "maciekbanas", + "name" = "Maciej Banas", + "state" = "active", + "locked" = FALSE, + "avatar_url" = "", + "web_url" = "" + ) +) diff --git a/tests/testthat/test-get_commits-GitLab.R b/tests/testthat/test-get_commits-GitLab.R index 7a55e381..b0113ab7 100644 --- a/tests/testthat/test-get_commits-GitLab.R +++ b/tests/testthat/test-get_commits-GitLab.R @@ -39,11 +39,46 @@ test_that("`prepare_commits_table()` prepares table of commits properly", { test_mocker$cache(gl_commits_table) }) +test_that("get_authors_dict() prepares dictionary with handles and names", { + mockery::stub( + test_rest_gitlab_priv$get_authors_dict, + "self$response", + test_fixtures$gitlab_user_search_response + ) + authors_dict <- test_rest_gitlab_priv$get_authors_dict( + commits_table = test_mocker$use("gl_commits_table"), + progress = FALSE + ) + expect_s3_class(authors_dict, "data.frame") + expect_true(nrow(authors_dict) > 0) + expect_length(authors_dict, 3) + test_mocker$cache(authors_dict) +}) + +test_that("`get_commits_authors_handles_and_names()` adds author logis and names to commits table", { + mockery::stub( + test_rest_gitlab$get_commits_authors_handles_and_names, + "private$get_authors_dict", + test_mocker$use("authors_dict") + ) + expect_snapshot( + gl_commits_table <- test_rest_gitlab$get_commits_authors_handles_and_names( + commits_table = test_mocker$use("gl_commits_table"), + verbose = TRUE + ) + ) + expect_commits_table( + gl_commits_table, + exp_auth = TRUE + ) + test_mocker$cache(gl_commits_table) +}) + test_that("get_commits_from_orgs works", { mockery::stub( gitlab_testhost_priv$get_commits_from_orgs, - "rest_engine$get_commits_from_repos", - test_mocker$use("gl_commits_org") + "rest_engine$get_commits_authors_handles_and_names", + test_mocker$use("gl_commits_table") ) suppressMessages( gl_commits_table <- gitlab_testhost_priv$get_commits_from_orgs( @@ -74,17 +109,3 @@ test_that("get_commits for GitLab works with repos implied", { gl_commits_table ) }) - -test_that("`get_commits_authors_handles_and_names()` adds author logis and names to commits table", { - expect_snapshot( - gl_commits_table <- test_rest_gitlab$get_commits_authors_handles_and_names( - commits_table = test_mocker$use("gl_commits_table"), - verbose = TRUE - ) - ) - expect_commits_table( - gl_commits_table, - exp_auth = TRUE - ) - test_mocker$cache(gl_commits_table) -}) diff --git a/tests/testthat/test-get_urls_repos-GitHub.R b/tests/testthat/test-get_urls_repos-GitHub.R index 49bb1e77..f211f049 100644 --- a/tests/testthat/test-get_urls_repos-GitHub.R +++ b/tests/testthat/test-get_urls_repos-GitHub.R @@ -1,4 +1,9 @@ test_that("get_repos_urls() works", { + mockery::stub( + test_rest_github$get_repos_urls, + "self$response", + test_fixtures$github_repositories_rest_response + ) gh_repos_urls <- test_rest_github$get_repos_urls( type = "web", org = "r-world-devs" @@ -13,6 +18,11 @@ test_that("get_repos_urls() works", { test_that("get_all_repos_urls prepares api repo_urls vector", { github_testhost_priv <- create_github_testhost(orgs = c("r-world-devs", "openpharma"), mode = "private") + mockery::stub( + test_rest_github$get_repos_urls, + "self$response", + test_fixtures$github_repositories_rest_response + ) gh_api_repos_urls <- github_testhost_priv$get_all_repos_urls( type = "api", verbose = FALSE diff --git a/tests/testthat/test-get_urls_repos-GitLab.R b/tests/testthat/test-get_urls_repos-GitLab.R index f97970c3..5e743c0b 100644 --- a/tests/testthat/test-get_urls_repos-GitLab.R +++ b/tests/testthat/test-get_urls_repos-GitLab.R @@ -1,20 +1,34 @@ test_that("get_repos_urls() works", { - gl_repos_urls <- test_rest_gitlab$get_repos_urls( + mockery::stub( + test_rest_gitlab$get_repos_urls, + "self$response", + test_fixtures$gitlab_repositories_rest_response + ) + gl_api_repos_urls <- test_rest_gitlab$get_repos_urls( type = "api", org = "mbtests" ) expect_gt( - length(gl_repos_urls), + length(gl_api_repos_urls), + 0 + ) + test_mocker$cache(gl_api_repos_urls) + gl_web_repos_urls <- test_rest_gitlab$get_repos_urls( + type = "web", + org = "mbtests" + ) + expect_gt( + length(gl_web_repos_urls), 0 ) - test_mocker$cache(gl_repos_urls) + test_mocker$cache(gl_web_repos_urls) }) test_that("get_all_repos_urls prepares api repo_urls vector", { mockery::stub( gitlab_testhost_priv$get_all_repos_urls, "rest_engine$get_repos_urls", - test_mocker$use("gl_repos_urls") + test_mocker$use("gl_api_repos_urls") ) gl_api_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( type = "api", @@ -23,9 +37,25 @@ test_that("get_all_repos_urls prepares api repo_urls vector", { expect_gt(length(gl_api_repos_urls), 0) expect_true(all(grepl("api", gl_api_repos_urls))) test_mocker$cache(gl_api_repos_urls) + mockery::stub( + gitlab_testhost_priv$get_all_repos_urls, + "rest_engine$get_repos_urls", + test_mocker$use("gl_web_repos_urls") + ) + gl_web_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( + type = "web", + verbose = FALSE + ) + expect_gt(length(gl_web_repos_urls), 0) + test_mocker$cache(gl_web_repos_urls) }) test_that("get_all_repos_urls prepares web repo_urls vector", { + mockery::stub( + gitlab_testhost_priv$get_all_repos_urls, + "rest_engine$get_repos_urls", + test_mocker$use("gl_web_repos_urls") + ) gl_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( type = "web", verbose = FALSE diff --git a/tests/testthat/test-get_user-GitHub.R b/tests/testthat/test-get_user-GitHub.R index a54d6bb6..6638be43 100644 --- a/tests/testthat/test-get_user-GitHub.R +++ b/tests/testthat/test-get_user-GitHub.R @@ -7,24 +7,13 @@ test_that("user query is built properly", { test_mocker$cache(gh_user_query) }) -test_that("`gql_response()` work as expected for GitHub", { - gh_user_gql_response <- test_graphql_github$gql_response( - test_mocker$use("gh_user_query"), - vars = list(user = "maciekbanas") - ) - expect_user_gql_response( - gh_user_gql_response - ) - test_mocker$cache(gh_user_gql_response) -}) - test_that("get_user pulls GitHub user response", { mockery::stub( test_graphql_github$get_user, "self$gql_response", - test_mocker$use("gh_user_gql_response") + test_fixtures$github_user_response ) - gh_user_response <- test_graphql_github$get_user(username = "maciekbanas") + gh_user_response <- test_graphql_github$get_user(username = "testuser") expect_user_gql_response( gh_user_response ) @@ -43,8 +32,13 @@ test_that("GitHub prepares user table", { }) test_that("GitHost gets users tables", { + mockery::stub( + github_testhost$get_users, + "private$prepare_user_table", + test_mocker$use("gh_user_table") + ) users_table <- github_testhost$get_users( - users = c("maciekbanas", "kalimu", "galachad") + users = c("testuser1", "testuser2") ) expect_users_table(users_table) test_mocker$cache(users_table) diff --git a/tests/testthat/test-get_user-GitLab.R b/tests/testthat/test-get_user-GitLab.R index 302cb4db..726b5bc7 100644 --- a/tests/testthat/test-get_user-GitLab.R +++ b/tests/testthat/test-get_user-GitLab.R @@ -7,25 +7,13 @@ test_that("user query is built properly", { test_mocker$cache(gl_user_query) }) -test_that("`gql_response()` work as expected for GitLab", { - gl_user_gql_response <- test_graphql_gitlab$gql_response( - test_mocker$use("gl_user_query"), - vars = list(user = "maciekbanas") - ) - expect_user_gql_response( - gl_user_gql_response - ) - test_mocker$cache(gl_user_gql_response) -}) - - test_that("get_user pulls GitLab user response", { mockery::stub( test_graphql_gitlab$get_user, "self$gql_response", - test_mocker$use("gl_user_gql_response") + test_fixtures$gitlab_user_response ) - gl_user_response <- test_graphql_gitlab$get_user(username = "maciekbanas") + gl_user_response <- test_graphql_gitlab$get_user(username = "testuser") expect_user_gql_response( gl_user_response ) @@ -43,18 +31,14 @@ test_that("GitLab prepares user table", { test_mocker$cache(gl_user_table) }) -test_that("get_users build users table for GitHub", { - users_result <- gitlab_testhost$get_users( - users = c("maciekbanas", "Cotau", "marcinkowskak") - ) - expect_users_table( - users_result - ) -}) - test_that("get_users build users table for GitLab", { + mockery::stub( + gitlab_testhost$get_users, + "private$prepare_user_table", + test_mocker$use("gl_user_table") + ) users_result <- gitlab_testhost$get_users( - users = c("maciekbanas", "Cotau", "marcinkowskak") + users = c("testuser1", "testuser2") ) expect_users_table( users_result From af204139a988decd68ffe0aa457dcd93ef843703 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 7 Oct 2024 09:48:38 +0000 Subject: [PATCH 06/94] Mocking API responses in tests, improve test objects, make more use of them, small improvements to code. --- R/EngineGraphQL.R | 4 +- R/EngineGraphQLGitHub.R | 64 +++++---- R/EngineRest.R | 2 +- R/EngineRestGitHub.R | 2 +- R/EngineRestGitLab.R | 2 +- R/GitHost.R | 7 +- R/test_helpers.R | 28 ++-- ...repos-GitHub.md => 01-get_repos-GitHub.md} | 2 +- ...repos-GitLab.md => 01-get_repos-GitLab.md} | 0 ...s-GitStats.md => 01-get_repos-GitStats.md} | 0 .../_snaps/get_files_structure-GitHub.md | 9 -- tests/testthat/_snaps/z-GitStats.md | 2 +- tests/testthat/helper-fixtures.R | 118 +++++++++++++++-- tests/testthat/helper.R | 29 ++-- tests/testthat/setup.R | 117 ++++++++--------- ...-api-requests.R => test-00-api-requests.R} | 6 +- ...os-GitHub.R => test-01-get_repos-GitHub.R} | 124 ++++++++++++------ ...os-GitLab.R => test-01-get_repos-GitLab.R} | 0 ...itStats.R => test-01-get_repos-GitStats.R} | 10 -- .../testthat/test-get_files_content-GitHub.R | 111 ++++++---------- .../testthat/test-get_files_content-GitLab.R | 25 +++- .../test-get_files_structure-GitHub.R | 86 +++++++----- .../test-get_files_structure-GitStats.R | 4 +- tests/testthat/test-get_release-GitHub.R | 17 +++ tests/testthat/test-get_urls_repos-GitHub.R | 21 ++- tests/testthat/test-get_urls_repos-GitStats.R | 2 +- tests/testthat/test-get_user-GitHub.R | 6 +- tests/testthat/test-get_user-GitLab.R | 5 +- tests/testthat/test-get_users-GitStats.R | 18 ++- tests/testthat/test-helpers.R | 18 +++ tests/testthat/test-show_orgs.R | 2 +- tests/testthat/test-z-GitStats.R | 2 +- 32 files changed, 513 insertions(+), 330 deletions(-) rename tests/testthat/_snaps/{get_repos-GitHub.md => 01-get_repos-GitHub.md} (95%) rename tests/testthat/_snaps/{get_repos-GitLab.md => 01-get_repos-GitLab.md} (100%) rename tests/testthat/_snaps/{get_repos-GitStats.md => 01-get_repos-GitStats.md} (100%) rename tests/testthat/{test-api-requests.R => test-00-api-requests.R} (89%) rename tests/testthat/{test-get_repos-GitHub.R => test-01-get_repos-GitHub.R} (75%) rename tests/testthat/{test-get_repos-GitLab.R => test-01-get_repos-GitLab.R} (100%) rename tests/testthat/{test-get_repos-GitStats.R => test-01-get_repos-GitStats.R} (86%) diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R index aff83114..42f314e3 100644 --- a/R/EngineGraphQL.R +++ b/R/EngineGraphQL.R @@ -50,9 +50,9 @@ EngineGraphQL <- R6::R6Class( private = list( # GraphQL method for pulling response from API - perform_request = function(gql_query, vars) { + perform_request = function(gql_query, vars, token = private$token) { response <- httr2::request(paste0(self$gql_api_url, "?")) %>% - httr2::req_headers("Authorization" = paste0("Bearer ", private$token)) %>% + httr2::req_headers("Authorization" = paste0("Bearer ", token)) %>% httr2::req_body_json(list(query = gql_query, variables = vars)) %>% httr2::req_retry( is_transient = ~ httr2::resp_status(.x) %in% c(400, 502), diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index e837945a..80f8eb93 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -99,30 +99,14 @@ EngineGraphQLGitHub <- R6::R6Class( ) repositories <- repo_data[["repositories"]] def_branches <- repo_data[["def_branches"]] - org_files_list <- purrr::map2(repositories, def_branches, function(repo, def_branch) { - if (!is.null(host_files_structure)) { - file_paths <- private$get_path_from_files_structure( - host_files_structure = host_files_structure, - only_text_files = only_text_files, - org = org, - repo = repo - ) - } else if (is.null(host_files_structure) && only_text_files) { - file_paths <- file_paths[!grepl(non_text_files_pattern, file_paths)] - } - repo_files_list <- purrr::map(file_paths, function(file_path) { - private$get_file_response( - org = org, - repo = repo, - def_branch = def_branch, - file_path = file_path, - files_query = self$gql_query$file_blob_from_repo() - ) - }) %>% - purrr::map(~ .$data$repository) - names(repo_files_list) <- file_paths - return(repo_files_list) - }, .progress = progress) + org_files_list <- private$get_repositories_with_files( + repositories = repositories, + def_branches = def_branches, + file_paths = file_paths, + host_files_structure = host_files_structure, + only_text_files = only_text_files, + progress = progress + ) names(org_files_list) <- repositories for (file_path in file_paths) { org_files_list <- purrr::discard(org_files_list, ~ length(.[[file_path]]$file) == 0) @@ -254,6 +238,38 @@ EngineGraphQLGitHub <- R6::R6Class( return(result) }, + get_repositories_with_files = function(repositories, + def_branches, + host_files_structure, + only_text_files, + file_paths, + progress) { + purrr::map2(repositories, def_branches, function(repo, def_branch) { + if (!is.null(host_files_structure)) { + file_paths <- private$get_path_from_files_structure( + host_files_structure = host_files_structure, + only_text_files = only_text_files, + org = org, + repo = repo + ) + } else if (is.null(host_files_structure) && only_text_files) { + file_paths <- file_paths[!grepl(non_text_files_pattern, file_paths)] + } + repo_files_list <- purrr::map(file_paths, function(file_path) { + private$get_file_response( + org = org, + repo = repo, + def_branch = def_branch, + file_path = file_path, + files_query = self$gql_query$file_blob_from_repo() + ) + }) %>% + purrr::map(~ .$data$repository) + names(repo_files_list) <- file_paths + return(repo_files_list) + }, .progress = progress) + }, + get_file_response = function(org, repo, def_branch, file_path, files_query) { expression <- paste0(def_branch, ":", file_path) files_response <- self$gql_response( diff --git a/R/EngineRest.R b/R/EngineRest.R index deab2010..d20ab0a5 100644 --- a/R/EngineRest.R +++ b/R/EngineRest.R @@ -42,7 +42,7 @@ EngineRest <- R6::R6Class("EngineRest", private = list( # Paginate contributors and parse response into character vector - pull_contributors_from_repo = function(contributors_endpoint, user_name) { + get_contributors_from_repo = function(contributors_endpoint, user_name) { contributors_response <- private$paginate_results(contributors_endpoint) contributors_vec <- contributors_response %>% purrr::map_chr(~ eval(user_name)) %>% diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index b85b42a7..eccd0ca2 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -114,7 +114,7 @@ EngineRestGitHub <- R6::R6Class( repos_table$contributors <- purrr::map_chr(repo_iterator, function(repos_id) { tryCatch({ contributors_endpoint <- paste0(private$endpoints[["repositories"]], repos_id, "/contributors") - contributors_vec <- private$pull_contributors_from_repo( + contributors_vec <- private$get_contributors_from_repo( contributors_endpoint = contributors_endpoint, user_name = user_name ) diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 00b93618..86bf8209 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -113,7 +113,7 @@ EngineRestGitLab <- R6::R6Class( "/repository/contributors" ) contributors_vec <- tryCatch({ - private$pull_contributors_from_repo( + private$get_contributors_from_repo( contributors_endpoint = contributors_endpoint, user_name = user_name ) diff --git a/R/GitHost.R b/R/GitHost.R index a7b3169d..3b01b5fb 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -197,8 +197,10 @@ GitHost <- R6::R6Class( #' Iterator over pulling release logs from engines get_release_logs = function(since, until, verbose, progress) { - if (private$scan_all && is.null(private$orgs) && verbose) { - cli::cli_alert_info("[{private$host_name}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") + if (private$scan_all && is.null(private$orgs)) { + if (verbose) { + cli::cli_alert_info("[{private$host_name}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") + } private$orgs <- private$engines$graphql$get_orgs() } until <- until %||% Sys.time() @@ -668,7 +670,6 @@ GitHost <- R6::R6Class( information = "Pulling repositories (URLS)" ) } - repos <- private$set_repos(org) repos_urls <- rest_engine$get_repos_urls( type = type, org = org diff --git a/R/test_helpers.R b/R/test_helpers.R index d022630f..e582c54e 100644 --- a/R/test_helpers.R +++ b/R/test_helpers.R @@ -31,9 +31,10 @@ GitHostGitHubTest <- R6::R6Class( token = NA, host = NA) { private$set_api_url(host) + private$set_web_url(host) private$set_endpoints() private$check_if_public(host) - private$set_token(token, verbose = FALSE) + private$token <- token private$set_graphql_url() private$set_orgs_and_repos(orgs, repos, verbose = FALSE) private$setup_test_engines() @@ -66,9 +67,10 @@ GitHostGitLabTest <- R6::R6Class( token = NA, host = NA) { private$set_api_url(host) + private$set_web_url(host) private$set_endpoints() private$check_if_public(host) - private$set_token(token, verbose = FALSE) + private$token <- token private$set_graphql_url() private$set_orgs_and_repos(orgs, repos, verbose = FALSE) private$setup_test_engines() @@ -90,15 +92,16 @@ GitHostGitLabTest <- R6::R6Class( ) #' @noRd -create_github_testhost <- function(host = NULL, - orgs = NULL, +create_github_testhost <- function(host = NULL, + orgs = NULL, repos = NULL, + token = NULL, mode = "") { suppressMessages( test_host <- GitHostGitHubTest$new( - host = NULL, - token = Sys.getenv("GITHUB_PAT"), - orgs = orgs, + host = NULL, + token = token, + orgs = orgs, repos = repos ) ) @@ -109,15 +112,16 @@ create_github_testhost <- function(host = NULL, } #' @noRd -create_gitlab_testhost <- function(host = NULL, - orgs = NULL, +create_gitlab_testhost <- function(host = NULL, + orgs = NULL, repos = NULL, + token = NULL, mode = "") { suppressMessages( test_host <- GitHostGitLabTest$new( - host = NULL, - token = Sys.getenv("GITLAB_PAT_PUBLIC"), - orgs = orgs, + host = NULL, + token = token, + orgs = orgs, repos = repos ) ) diff --git a/tests/testthat/_snaps/get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md similarity index 95% rename from tests/testthat/_snaps/get_repos-GitHub.md rename to tests/testthat/_snaps/01-get_repos-GitHub.md index 904780b0..ed8eeb9b 100644 --- a/tests/testthat/_snaps/get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -18,7 +18,7 @@ Code gh_repos_table <- github_testhost_priv$get_all_repos() Message - i [Host:GitHub][Engine:GraphQl][Scope:r-world-devs] Pulling repositories... + i [Host:GitHub][Engine:GraphQl][Scope:test-org] Pulling repositories... # `get_repos_contributors()` adds contributors to repos table diff --git a/tests/testthat/_snaps/get_repos-GitLab.md b/tests/testthat/_snaps/01-get_repos-GitLab.md similarity index 100% rename from tests/testthat/_snaps/get_repos-GitLab.md rename to tests/testthat/_snaps/01-get_repos-GitLab.md diff --git a/tests/testthat/_snaps/get_repos-GitStats.md b/tests/testthat/_snaps/01-get_repos-GitStats.md similarity index 100% rename from tests/testthat/_snaps/get_repos-GitStats.md rename to tests/testthat/_snaps/01-get_repos-GitStats.md diff --git a/tests/testthat/_snaps/get_files_structure-GitHub.md b/tests/testthat/_snaps/get_files_structure-GitHub.md index 37945d6e..9a747432 100644 --- a/tests/testthat/_snaps/get_files_structure-GitHub.md +++ b/tests/testthat/_snaps/get_files_structure-GitHub.md @@ -5,15 +5,6 @@ Output [1] "query GetFilesFromRepo($org: String!, $repo: String!, $expression: String!) {\n repository(owner: $org, name: $repo) {\n id\n name\n url\n object(expression: $expression) {\n ... on Tree {\n entries {\n name\n type\n }\n }\n }\n }\n }" -# get_files_structure_from_orgs pulls files structure for repositories in orgs - - Code - gh_files_structure_from_orgs <- github_testhost_priv$ - get_files_structure_from_orgs(pattern = NULL, depth = 2L, verbose = TRUE) - Message - i [Host:GitHub][Engine:GraphQl][Scope:r-world-devs] Pulling files structure...... - i [Host:GitHub][Engine:GraphQl][Scope:openpharma] Pulling files structure...... - # when files_structure is empty, appropriate message is returned Code diff --git a/tests/testthat/_snaps/z-GitStats.md b/tests/testthat/_snaps/z-GitStats.md index a460e5d0..ffce6da0 100644 --- a/tests/testthat/_snaps/z-GitStats.md +++ b/tests/testthat/_snaps/z-GitStats.md @@ -18,7 +18,7 @@ A GitStats object for 2 hosts: Hosts: https://api.github.com, https://gitlab.com/api/v4 Scanning scope: - Organizations: [2] r-world-devs, mbtests + Organizations: [2] github_test_org, gitlab_test_group Repositories: [0] Storage: diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 8be5d657..f9be55e5 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -30,14 +30,43 @@ test_fixtures$half_empty_gql_response <- list( ) ) -test_fixtures$github_repositories_rest_response <- list( - list( - "html_url" = "https://github.com/r-world-devs/GitStats", - "url" = "https://api.github.com/repos/r-world-devs/GitStats" +test_fixtures$github_repository_rest_response <- list( + "id" = 627452680, + "node_id" = "R_kgDOJWYrCA", + "name" = "testRepo", + "full_name" = "test-org/TestRepo", + "private" = FALSE, + "owner" = list( + "login" = "test-org", + "id" = 103638913, + "node_id" = "O_kgDOBi1ngQ", + "avatar_url" = "https://avatars.githubusercontent.com/u/103638913?v=4" ), + "html_url" = "https://testhost.com/test-org/TestRepo", + "description" = NULL, + "fork" = FALSE, + "url" = "https://testhost.com/api/v4/repos/test-org/TestRepo", + "forks_url" = "https://testhost.com/api/v4/repos/test-org/TestRepo/forks", + "collaborators_url" = "https://testhost.com/api/v4/repos/test-org/TestRepo/collaborators{/collaborator}", + "teams_url" = "https://testhost.com/api/v4/repos/test-org/TestRepo/teams", + "events_url" = "https://testhost.com/api/v4/repos/test-org/TestRepo/events", + "branches_url" = "https://testhost.com/api/v4/repos/test-org/TestRepo/branches{/branch}", + "created_at" = "2023-04-13T13:52:24Z", + "pushed_at" = "2023-12-21T20:36:23Z", + "size" = 211L, + "stargazers_count" = 2, + "watchers_count" = 2, + "language" = "R", + "forks_count" = 0, + "open_issues_count" = 3L +) + +test_fixtures$github_repositories_rest_response <- list( + test_fixtures$github_repository_rest_response, list( - "html_url" = "https://github.com/openpharma/DataFakeR", - "url" = "https://api.github.com/repos/openpharma/DataFakeR" + "id" = 2222222222222, + "html_url" = "https://testhost.com/test-org/TestRepo", + "url" = "https://testhost.com/api/v4/repos/test-org/TestRepo" ) ) @@ -378,7 +407,7 @@ test_fixtures$github_search_response <- list( "repository" = list( "id" = 627452680, "url" = "https://api.github.com/repos/r-world-devs/GitStats", - "html" = "https://github.com/r-world-devs/GitStats" + "html_url" = "https://github.com/r-world-devs/GitStats" ), "score" = 1 ), @@ -392,7 +421,7 @@ test_fixtures$github_search_response <- list( "repository" = list( "id" = 604718884, "url" = "https://api.github.com/repos/r-world-devs/GitStats", - "html" = "https://github.com/r-world-devs/GitStats" + "html_url" = "https://github.com/r-world-devs/GitStats" ), "score" = 1 ), @@ -406,7 +435,7 @@ test_fixtures$github_search_response <- list( "repository" = list( "id" = 495151911, "url" = "https://api.github.com/repos/r-world-devs/GitStats", - "html" = "https://github.com/r-world-devs/GitStats" + "html_url" = "https://github.com/r-world-devs/GitStats" ), "score" = 1 ) @@ -451,8 +480,8 @@ test_fixtures$github_files_tree_response <- list( "data" = list( "repository" = list( "id" = "R_kgD0Ivtxsg", - "name" = "GitStats", - "url" = "https://github.com/r-world-devs/GitStats", + "name" = "TestRepo", + "url" = "https://github.com/test-org/TestRepo", "object" = list( "entries" = list( list( @@ -471,6 +500,18 @@ test_fixtures$github_files_tree_response <- list( "name" = ".covrignore", "type" = "blob" ), + list( + "name" = "test.md", + "type" = "blob" + ), + list( + "name" = "test.qmd", + "type" = "blob" + ), + list( + "name" = "test.Rmd", + "type" = "blob" + ), list( "name" = "renv", "type" = "tree" @@ -636,3 +677,58 @@ test_fixtures$gitlab_user_search_response <- list( "web_url" = "" ) ) + +test_fixtures$github_contributor_response <- list( + "login" = "test_login", + "id" = 11111L, + "node_id" = "xxxxx", + "avatar_url" = "", + "gravatar_id" = "", + "url" = "", + "html_url" = "", + "followers_url" = "" +) + +test_fixtures$github_contributors_response <- list( + test_fixtures$github_contributor_response, + test_fixtures$github_contributor_response, + test_fixtures$github_contributor_response +) + +test_fixtures$github_open_issue_response <- list( + "url" = "test", + "repository_url" = "test", + "labels_url" = "test", + "comments_url" = "test", + "events_url" = "test", + "html_url" = "test", + "id" = 111111L, + "node_id" = "xxxxxx", + "number" = 1L, + "title" = "Test title", + "user" = list(), + "labels" = list(), + "state" = "open" +) + +test_fixtures$github_closed_issue_response <- list( + "url" = "test", + "repository_url" = "test", + "labels_url" = "test", + "comments_url" = "test", + "events_url" = "test", + "html_url" = "test", + "id" = 111111L, + "node_id" = "xxxxxx", + "number" = 1L, + "title" = "Test title", + "user" = list(), + "labels" = list(), + "state" = "closed" +) + +test_fixtures$github_issues_response <- list( + test_fixtures$github_open_issue_response, + test_fixtures$github_open_issue_response, + test_fixtures$github_closed_issue_response +) diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R index d8f5d027..a749d565 100644 --- a/tests/testthat/helper.R +++ b/tests/testthat/helper.R @@ -8,28 +8,17 @@ create_test_gitstats <- function(hosts = 0, inject_users = NULL, inject_package_usage = NULL) { test_gitstats <- create_gitstats() - if (hosts == 1) { - suppressMessages({ - test_gitstats$set_github_host( - host = NULL, - token = Sys.getenv("GITHUB_PAT"), - orgs = c("r-world-devs", "openpharma") - ) - }) + test_gitstats$.__enclos_env__$private$hosts[[1]] <- create_github_testhost( + orgs = "test_org" + ) } else if (hosts == 2) { - suppressMessages({ - test_gitstats$set_github_host( - host = NULL, - token = Sys.getenv("GITHUB_PAT"), - orgs = c("r-world-devs") - ) - test_gitstats$set_gitlab_host( - host = NULL, - token = Sys.getenv("GITLAB_PAT_PUBLIC"), - orgs = "mbtests" - ) - }) + test_gitstats$.__enclos_env__$private$hosts[[1]] <- create_github_testhost( + orgs = "github_test_org" + ) + test_gitstats$.__enclos_env__$private$hosts[[2]] <- create_gitlab_testhost( + orgs = "gitlab_test_group" + ) } if (!is.null(inject_repos)) { test_gitstats$.__enclos_env__$private$storage$repositories <- test_mocker$use(inject_repos) diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 235256a8..0b13ecd4 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -1,66 +1,3 @@ -test_mocker <- Mocker$new() - -test_gitstats <- create_test_gitstats(hosts = 2) -test_gitstats_priv <- create_test_gitstats(hosts = 2, priv_mode = TRUE) - -test_gqlquery_gh <- GQLQueryGitHub$new() -test_gqlquery_gl <- GQLQueryGitLab$new() - -test_rest_github <- EngineRestGitHub$new( - rest_api_url = "https://api.github.com", - token = Sys.getenv("GITHUB_PAT") -) -test_rest_github_priv <- environment(test_rest_github$initialize)$private - -test_graphql_github <- EngineGraphQLGitHub$new( - gql_api_url = "https://api.github.com/graphql", - token = Sys.getenv("GITHUB_PAT") -) -test_graphql_github_priv <- environment(test_graphql_github$initialize)$private - -test_rest_gitlab <- EngineRestGitLab$new( - rest_api_url = "https://gitlab.com/api/v4", - token = Sys.getenv("GITLAB_PAT_PUBLIC") -) -test_rest_gitlab_priv <- environment(test_rest_gitlab$initialize)$private - -test_graphql_gitlab <- EngineGraphQLGitLab$new( - gql_api_url = "https://gitlab.com/api/graphql", - token = Sys.getenv("GITLAB_PAT_PUBLIC") -) -test_graphql_gitlab_priv <- environment(test_graphql_gitlab$initialize)$private - -github_testhost <- create_github_testhost(orgs = "r-world-devs") - -github_testhost_priv <- create_github_testhost(orgs = "r-world-devs", mode = "private") - -github_testhost_repos <- create_github_testhost( - repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder") -) - -github_testhost_repos_priv <- create_github_testhost( - repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder"), - mode = "private" -) - -gitlab_testhost <- create_gitlab_testhost(orgs = "mbtests") - -gitlab_testhost_priv <- create_gitlab_testhost(orgs = "mbtests", mode = "private") - -gitlab_testhost_repos <- create_gitlab_testhost( - repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2") -) - -test_settings <- list( - verbose = TRUE, - storage = TRUE -) - -test_settings_silence <- list( - verbose = FALSE, - storage = TRUE -) - if (nchar(Sys.getenv("GITHUB_PAT")) == 0) { cli::cli_abort(c( "x" = "You did not set up your GITHUB_PAT environment variable.", @@ -110,3 +47,57 @@ if (nchar(Sys.getenv("GITLAB_PAT_PUBLIC")) > 0) { } }) } + +test_mocker <- Mocker$new() + +test_gitstats <- create_test_gitstats(hosts = 2) +test_gitstats_priv <- create_test_gitstats(hosts = 2, priv_mode = TRUE) + +test_gqlquery_gh <- GQLQueryGitHub$new() +test_gqlquery_gl <- GQLQueryGitLab$new() + +test_rest_github <- TestEngineRestGitHub$new( + rest_api_url = "https://api.github.com", + token = NULL +) +test_rest_github_priv <- environment(test_rest_github$initialize)$private + +test_graphql_github <- EngineGraphQLGitHub$new( + gql_api_url = "https://api.github.com/graphql", + token = NULL +) +test_graphql_github_priv <- environment(test_graphql_github$initialize)$private + +test_rest_gitlab <- TestEngineRestGitLab$new( + rest_api_url = "https://gitlab.com/api/v4", + token = NULL +) +test_rest_gitlab_priv <- environment(test_rest_gitlab$initialize)$private + +test_graphql_gitlab <- EngineGraphQLGitLab$new( + gql_api_url = "https://gitlab.com/api/graphql", + token = NULL +) +test_graphql_gitlab_priv <- environment(test_graphql_gitlab$initialize)$private + +github_testhost <- create_github_testhost(orgs = "test-org") + +github_testhost_priv <- create_github_testhost(orgs = "test-org", mode = "private") + +github_testhost_repos <- create_github_testhost( + repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder") +) + +github_testhost_repos_priv <- create_github_testhost( + repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder"), + mode = "private" +) + +gitlab_testhost <- create_gitlab_testhost(orgs = "mbtests") + +gitlab_testhost_priv <- create_gitlab_testhost(orgs = "mbtests", mode = "private") + +gitlab_testhost_repos <- create_gitlab_testhost( + repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2") +) + diff --git a/tests/testthat/test-api-requests.R b/tests/testthat/test-00-api-requests.R similarity index 89% rename from tests/testthat/test-api-requests.R rename to tests/testthat/test-00-api-requests.R index 59db3b48..8ebf8117 100644 --- a/tests/testthat/test-api-requests.R +++ b/tests/testthat/test-00-api-requests.R @@ -26,6 +26,7 @@ test_that("`perform_request()` throws error on bad host", { }) test_that("`perform_request()` returns proper status", { + skip_if(Sys.getenv("GITHUB_PAT") == "") bad_endpoint <- "https://api.github.com/orgs/everybody_loves_somebody" expect_error( test_rest_github_priv$perform_request( @@ -37,6 +38,7 @@ test_that("`perform_request()` returns proper status", { }) test_that("`perform_request()` returns status 200", { + skip_if(Sys.getenv("GITHUB_PAT") == "") response <- test_rest_github_priv$perform_request( endpoint = "https://api.github.com/repos/r-world-devs/GitStats", token = Sys.getenv("GITHUB_PAT") @@ -48,13 +50,15 @@ test_that("`perform_request()` returns status 200", { }) test_that("`perform_request()` for GraphQL returns status 200", { + skip_if(Sys.getenv("GITHUB_PAT") == "") response <- test_graphql_github_priv$perform_request( gql_query = "{ viewer { login } }", - vars = NULL + vars = NULL, + token = Sys.getenv("GITHUB_PAT") ) expect_equal( response$status_code, diff --git a/tests/testthat/test-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R similarity index 75% rename from tests/testthat/test-get_repos-GitHub.R rename to tests/testthat/test-01-get_repos-GitHub.R index 5e641963..4dc39f81 100644 --- a/tests/testthat/test-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -44,25 +44,12 @@ test_that("`get_repos_from_org()` prepares formatted list", { # REST Engine search repos by code -test_that("`response()` returns search response from GitHub's REST API", { - gh_search_response_raw <- test_fixtures$github_search_response - expect_gh_search_response(gh_search_response_raw[["items"]]) - test_mocker$cache(gh_search_response_raw) -}) - -test_that("`response()` returns search response from GitHub's REST API", { - search_endpoint <- "https://api.github.com/search/code?q=shiny+user:openpharma+in:file+filename:DESCRIPTION" - gh_search_response_in_file <- test_rest_github$response(search_endpoint)[["items"]] - expect_gh_search_response(gh_search_response_in_file) - test_mocker$cache(gh_search_response_in_file) -}) - test_that("`search_response()` performs search with limit under 100", { - total_n <- test_mocker$use("gh_search_response_raw")[["total_count"]] + total_n <- test_fixtures$github_search_response[["total_count"]] mockery::stub( test_rest_github_priv$search_response, "self$response", - test_mocker$use("gh_search_response_raw") + test_fixtures$github_search_response ) gh_search_repos_response <- test_rest_github_priv$search_response( search_endpoint = test_mocker$use("search_endpoint"), @@ -74,42 +61,68 @@ test_that("`search_response()` performs search with limit under 100", { }) test_that("Mapping search result to repositories works", { - result <- test_rest_github_priv$map_search_into_repos( + mockery::stub( + test_rest_github_priv$map_search_into_repos, + "self$response", + test_fixtures$github_repository_rest_response + ) + gh_mapped_repos <- test_rest_github_priv$map_search_into_repos( search_response = test_mocker$use("gh_search_repos_response"), progress = FALSE ) - expect_gh_repos_rest_response(result) + expect_gh_repos_rest_response(gh_mapped_repos) + test_mocker$cache(gh_mapped_repos) }) test_that("`get_repos_by_code()` returns repos output for code search in files", { + mockery::stub( + test_rest_github$get_repos_by_code, + "self$response", + list("total_count" = 3L) + ) mockery::stub( test_rest_github$get_repos_by_code, "private$search_response", - test_mocker$use("gh_search_response_in_file") + test_fixtures$github_search_response + ) + mockery::stub( + test_rest_github$get_repos_by_code, + "private$map_search_into_repos", + test_mocker$use("gh_mapped_repos") ) gh_repos_by_code <- test_rest_github$get_repos_by_code( - code = "shiny", - filename = "DESCRIPTION", - org = "openpharma", - verbose = FALSE + code = "test_code", + filename = "test_file", + org = "test_org", + verbose = FALSE ) expect_gh_repos_rest_response(gh_repos_by_code) test_mocker$cache(gh_repos_by_code) }) test_that("`get_repos_by_code()` for GitHub prepares a raw (raw_output = TRUE) search response", { + mockery::stub( + test_rest_github$get_repos_by_code, + "self$response", + list("total_count" = 3L) + ) mockery::stub( test_rest_github$get_repos_by_code, "private$search_response", - test_mocker$use("gh_search_repos_response") + test_fixtures$github_search_response + ) + mockery::stub( + test_rest_github$get_repos_by_code, + "private$map_search_into_repos", + test_mocker$use("gh_mapped_repos") ) gh_repos_by_code_raw <- test_rest_github$get_repos_by_code( - code = "shiny", - org = "openpharma", + code = "test_code", + org = "test_org", raw_output = TRUE, - verbose = FALSE + verbose = FALSE ) - expect_gh_search_response(gh_repos_by_code_raw) + expect_gh_search_response(gh_repos_by_code_raw$items) test_mocker$cache(gh_repos_by_code_raw) }) @@ -148,6 +161,30 @@ test_that("`prepare_repos_table()` prepares repos table", { test_mocker$cache(gh_repos_by_code_table) }) +test_that("`get_repos_issues()` adds issues to repos table", { + mockery::stub( + test_rest_github$get_repos_issues, + "self$response", + test_fixtures$github_issues_response + ) + gh_repos_by_code_table <- test_mocker$use("gh_repos_by_code_table") + suppressMessages( + gh_repos_by_code_table <- test_rest_github$get_repos_issues( + gh_repos_by_code_table, + progress = FALSE + ) + ) + expect_gt( + length(gh_repos_by_code_table$issues_open), + 0 + ) + expect_gt( + length(gh_repos_by_code_table$issues_closed), + 0 + ) + test_mocker$cache(gh_repos_by_code_table) +}) + test_that("`get_repos_with_code_from_orgs()` works", { mockery::stub( github_testhost_priv$get_repos_with_code_from_orgs, @@ -156,7 +193,7 @@ test_that("`get_repos_with_code_from_orgs()` works", { ) mockery::stub( github_testhost_priv$get_repos_with_code_from_orgs, - "private$prepare_repos_table_from_rest", + "rest_engine$get_repos_issues", test_mocker$use("gh_repos_by_code_table") ) repos_with_code <- github_testhost_priv$get_repos_with_code_from_orgs( @@ -199,26 +236,27 @@ test_that("`get_all_repos()` works as expected", { test_mocker$cache(gh_repos_table) }) -test_that("`get_repos_issues()` adds issues to repos table", { - gh_repos_by_code_table <- test_mocker$use("gh_repos_by_code_table") - suppressMessages( - gh_repos_by_code_table <- test_rest_github$get_repos_issues( - gh_repos_by_code_table, - progress = FALSE - ) - ) - expect_gt( - length(gh_repos_by_code_table$issues_open), - 0 +test_that("get_contributors_from_repo", { + mockery::stub( + test_rest_github_priv$get_contributors_from_repo, + "private$paginate_results", + test_fixtures$github_contributors_response ) - expect_gt( - length(gh_repos_by_code_table$issues_closed), - 0 + github_contributors <- test_rest_github_priv$get_contributors_from_repo( + contributors_endpoint = "test_endpoint", + user_name = "test_user" ) - test_mocker$cache(gh_repos_by_code_table) + expect_type(github_contributors, "character") + expect_gt(length(github_contributors), 0) + test_mocker$cache(github_contributors) }) test_that("`get_repos_contributors()` adds contributors to repos table", { + mockery::stub( + test_rest_github$get_repos_contributors, + "private$get_contributors_from_repo", + test_mocker$use("github_contributors") + ) expect_snapshot( gh_repos_by_code_table <- test_rest_github$get_repos_contributors( repos_table = test_mocker$use("gh_repos_by_code_table"), diff --git a/tests/testthat/test-get_repos-GitLab.R b/tests/testthat/test-01-get_repos-GitLab.R similarity index 100% rename from tests/testthat/test-get_repos-GitLab.R rename to tests/testthat/test-01-get_repos-GitLab.R diff --git a/tests/testthat/test-get_repos-GitStats.R b/tests/testthat/test-01-get_repos-GitStats.R similarity index 86% rename from tests/testthat/test-get_repos-GitStats.R rename to tests/testthat/test-01-get_repos-GitStats.R index 74d55b7a..10fab878 100644 --- a/tests/testthat/test-get_repos-GitStats.R +++ b/tests/testthat/test-01-get_repos-GitStats.R @@ -79,13 +79,3 @@ test_that("get_repos works properly and for the second time uses cache", { with_cols = c("contributors", "contributors_n") ) }) - -test_that("get_repos pulls repositories without contributors", { - test_gitstats <- create_test_gitstats(hosts = 2) - repos_table <- test_gitstats$get_repos( - add_contributors = FALSE, - verbose = FALSE - ) - expect_repos_table(repos_table, repo_cols = repo_gitstats_colnames) - expect_false("contributors" %in% names(repos_table)) -}) diff --git a/tests/testthat/test-get_files_content-GitHub.R b/tests/testthat/test-get_files_content-GitHub.R index ae530e54..642e5235 100644 --- a/tests/testthat/test-get_files_content-GitHub.R +++ b/tests/testthat/test-get_files_content-GitHub.R @@ -8,20 +8,26 @@ test_that("file queries for GitHub are built properly", { }) test_that("get_repos_data pulls data on repos and branches", { - repos_data <- test_graphql_github_priv$get_repos_data( + mockery::stub( + test_graphql_github_priv$get_repos_data, + "self$get_repos_from_org", + test_mocker$use("gh_repos_from_org") + ) + gh_repos_data <- test_graphql_github_priv$get_repos_data( org = "r-world-devs", repos = NULL ) expect_equal( - names(repos_data), + names(gh_repos_data), c("repositories", "def_branches") ) expect_true( - length(repos_data$repositories) > 0 + length(gh_repos_data$repositories) > 0 ) expect_true( - length(repos_data$def_branches) > 0 + length(gh_repos_data$def_branches) > 0 ) + test_mocker$cache(gh_repos_data) }) test_that("GitHub GraphQL Engine pulls file response", { @@ -58,86 +64,46 @@ test_that("GitHub GraphQL Engine pulls png file response", { test_mocker$cache(github_png_file_response) }) -test_that("GitHub GraphQL Engine pulls files from organization", { +test_that("get_repositories_with_files works", { mockery::stub( - test_graphql_github$get_files_from_org, + test_graphql_github_priv$get_repositories_with_files, "private$get_file_response", test_mocker$use("github_file_response") ) - github_files_response <- test_graphql_github$get_files_from_org( - org = "r-world-devs", - repos = NULL, - file_paths = "LICENSE", + gh_repositories_with_files <- test_graphql_github_priv$get_repositories_with_files( + repositories = c("test_repo_1", "test_repo_2", "test_repo_3", "test_repo_4", "test_repo_5"), + def_branches = c("test_branch_1", "test_branch_2", "test_branch_3", "test_branch_4", "test_branch_5"), + file_paths = "test_files", only_text_files = TRUE, host_files_structure = NULL, - verbose = FALSE, - progress = FALSE - ) - expect_github_files_response(github_files_response) - test_mocker$cache(github_files_response) -}) - -test_that("GitHub GraphQL Engine pulls .png files from organization", { - mockery::stub( - test_graphql_github$get_files_from_org, - "private$get_file_response", - test_mocker$use("github_png_file_response") - ) - github_png_files_response <- test_graphql_github$get_files_from_org( - org = "r-world-devs", - repos = NULL, - file_paths = "man/figures/logo.png", - only_text_files = FALSE, - host_files_structure = NULL, - verbose = FALSE, progress = FALSE ) - expect_github_files_response(github_png_files_response) - test_mocker$cache(github_png_files_response) + expect_type(gh_repositories_with_files, "list") + test_mocker$cache(gh_repositories_with_files) }) -test_that("GitHub GraphQL Engine pulls files from defined repositories", { +test_that("GitHub GraphQL Engine pulls files from organization", { mockery::stub( test_graphql_github$get_files_from_org, - "private$get_file_response", - test_mocker$use("github_file_response") - ) - github_files_response <- test_graphql_github$get_files_from_org( - org = "openpharma", - repos = c("DataFakeR", "visR"), - file_paths = "README.md", - host_files_structure = NULL, - only_text_files = TRUE, - verbose = FALSE, - progress = FALSE + "private$get_repos_data", + test_mocker$use("gh_repos_data") ) - expect_github_files_response(github_files_response) - expect_equal(length(github_files_response), 2) -}) - -test_that("GitHub GraphQL Engine pulls two files from a group", { mockery::stub( test_graphql_github$get_files_from_org, - "private$get_file_response", - test_mocker$use("github_file_response") + "private$get_repositories_with_files", + test_mocker$use("gh_repositories_with_files") ) github_files_response <- test_graphql_github$get_files_from_org( - org = "r-world-devs", + org = "test_org", repos = NULL, - file_paths = c("DESCRIPTION", "NAMESPACE"), - host_files_structure = NULL, + file_paths = "test_files", only_text_files = TRUE, + host_files_structure = NULL, verbose = FALSE, progress = FALSE ) expect_github_files_response(github_files_response) - purrr::walk(github_files_response, ~ {expect_true( - all( - c("DESCRIPTION", "NAMESPACE") %in% - names(.) - ) - ) - }) + test_mocker$cache(github_files_response) }) test_that("GitHubHost prepares table from files response", { @@ -150,15 +116,20 @@ test_that("GitHubHost prepares table from files response", { test_mocker$cache(gh_files_table) }) -test_that("GitHubHost prepares table from png files (with no content) response", { - gh_png_files_table <- github_testhost_priv$prepare_files_table( - files_response = test_mocker$use("github_png_files_response"), - org = "r-world-devs", - file_path = "man/figures/logo.png" - ) - expect_files_table(gh_png_files_table) - expect_true(all(is.na(gh_png_files_table$file_content))) - test_mocker$cache(gh_png_files_table) +test_that("GitHubHost prepares table from files with no content", { + empty_files_response <- test_mocker$use("github_files_response") %>% + purrr::map(function(test_repo) { + test_repo$test_files$file$text <- NULL + return(test_repo) + }) + gh_empty_files_table <- github_testhost_priv$prepare_files_table( + files_response = empty_files_response, + org = "test_org", + file_path = "test_files" + ) + expect_files_table(gh_empty_files_table) + expect_true(all(is.na(gh_empty_files_table$file_content))) + test_mocker$cache(gh_empty_files_table) }) test_that("get_files_content_from_orgs for GitHub works", { diff --git a/tests/testthat/test-get_files_content-GitLab.R b/tests/testthat/test-get_files_content-GitLab.R index e3f5227d..9ea69b2e 100644 --- a/tests/testthat/test-get_files_content-GitLab.R +++ b/tests/testthat/test-get_files_content-GitLab.R @@ -29,16 +29,27 @@ test_that("get_file_blobs_response() works", { test_that("get_repos_data pulls data on repositories", { - repositories <- test_graphql_gitlab_priv$get_repos_data( + mockery::stub( + test_graphql_gitlab_priv$get_repos_data, + "self$get_repos_from_org", + test_mocker$use("gl_repos_from_org") + ) + gl_repos_data <- test_graphql_gitlab_priv$get_repos_data( org = "mbtests", repos = NULL ) expect_true( - length(repositories) > 0 + length(gl_repos_data) > 0 ) + test_mocker$cache(gl_repos_data) }) test_that("GitLab GraphQL Engine pulls files from a group", { + mockery::stub( + test_graphql_gitlab$get_files_from_org, + "self$get_repos_data", + test_mocker$use("gl_repos_data") + ) mockery::stub( test_graphql_gitlab$get_files_from_org, "self$gql_response", @@ -56,6 +67,11 @@ test_that("GitLab GraphQL Engine pulls files from a group", { }) test_that("GitLab GraphQL Engine pulls files from org by iterating over repos", { + mockery::stub( + test_graphql_gitlab$get_files_from_org_per_repo, + "self$get_repos_data", + test_mocker$use("gl_repos_data") + ) mockery::stub( test_graphql_gitlab$get_files_from_org_per_repo, "private$get_file_blobs_response", @@ -79,6 +95,11 @@ test_that("is query error is FALSE when response is empty (non query error)", { }) test_that("Gitlab GraphQL switches to pulling files per repositories when query is too complex", { + mockery::stub( + test_graphql_gitlab$get_files_from_org, + "self$get_repos_data", + test_mocker$use("gl_repos_data") + ) mockery::stub( test_graphql_gitlab$get_files_from_org, "private$is_query_error", diff --git a/tests/testthat/test-get_files_structure-GitHub.R b/tests/testthat/test-get_files_structure-GitHub.R index a7d52af9..0c63da20 100644 --- a/tests/testthat/test-get_files_structure-GitHub.R +++ b/tests/testthat/test-get_files_structure-GitHub.R @@ -14,8 +14,8 @@ test_that("get_file_response works", { test_fixtures$github_files_tree_response ) gh_files_tree_response <- test_graphql_github_priv$get_file_response( - org = "r-world-devs", - repo = "GitStats", + org = "test-org", + repo = "TestRepo", def_branch = "master", file_path = "", files_query = test_mocker$use("gh_files_tree_query") @@ -42,6 +42,11 @@ test_that("get_dirs_and_files returns list with directories and files", { }) test_that("get_files_structure_from_repo returns list with files and dirs vectors", { + mockery::stub( + test_graphql_github_priv$get_files_structure_from_repo, + "private$get_file_response", + test_mocker$use("gh_files_tree_response") + ) files_and_dirs <- test_mocker$use("files_and_dirs_list") files_and_dirs$dirs <- character() mockery::stub( @@ -50,8 +55,8 @@ test_that("get_files_structure_from_repo returns list with files and dirs vector files_and_dirs ) files_structure <- test_graphql_github_priv$get_files_structure_from_repo( - org = "r-world-devs", - repo = "GitStats", + org = "test-org", + repo = "TestRepo", def_branch = "master" ) expect_type( @@ -63,18 +68,23 @@ test_that("get_files_structure_from_repo returns list with files and dirs vector test_that("get_files_structure_from_repo returns list of files up to 2 tier of dirs", { mockery::stub( test_graphql_github_priv$get_files_structure_from_repo, - "private$get_files_tree_response", + "private$get_file_response", test_mocker$use("gh_files_tree_response") ) + mockery::stub( + test_graphql_github_priv$get_files_structure_from_repo, + "private$get_files_and_dirs", + files_and_dirs <- test_mocker$use("files_and_dirs_list") + ) files_structure_very_shallow <- test_graphql_github_priv$get_files_structure_from_repo( - org = "r-world-devs", - repo = "GitStats", + org = "test-org", + repo = "TestRepo", def_branch = "master", depth = 1L ) files_structure_shallow <- test_graphql_github_priv$get_files_structure_from_repo( - org = "r-world-devs", - repo = "GitStats", + org = "test-org", + repo = "TestRepo", def_branch = "master", depth = 2L ) @@ -95,6 +105,9 @@ test_that("only files with certain pattern are retrieved", { pattern = "\\.md|\\.qmd|\\.Rmd" ) files_structure <- test_mocker$use("files_structure") + expect_true( + length(md_files_structure) > 0 + ) expect_true( length(md_files_structure) < length(files_structure) ) @@ -102,59 +115,67 @@ test_that("only files with certain pattern are retrieved", { }) test_that("GitHub GraphQL Engine pulls files structure from repositories", { + mockery::stub( + test_graphql_github$get_files_structure_from_org, + "private$get_repos_data", + test_mocker$use("gh_repos_data") + ) mockery::stub( test_graphql_github$get_files_structure_from_org, "private$get_files_structure_from_repo", test_mocker$use("files_structure") ) gh_files_structure <- test_graphql_github$get_files_structure_from_org( - org = "r-world-devs", - repos = c("GitStats") + org = "test_org", + repos = rep("TestRepo", 5) ) purrr::walk(gh_files_structure, ~ expect_true(length(.) > 0)) expect_equal( names(gh_files_structure), - c("GitStats") + rep("TestRepo", 5) ) test_mocker$cache(gh_files_structure) }) test_that("GitHub GraphQL Engine pulls files structure with pattern from repositories", { + mockery::stub( + test_graphql_github$get_files_structure_from_org, + "private$get_repos_data", + test_mocker$use("gh_repos_data") + ) mockery::stub( test_graphql_github$get_files_structure_from_org, "private$get_files_structure_from_repo", test_mocker$use("md_files_structure") ) gh_md_files_structure <- test_graphql_github$get_files_structure_from_org( - org = "r-world-devs", - repos = "GitStats", + org = "test-org", + repos = "TestRepo", pattern = "\\.md|\\.qmd|\\.Rmd" ) purrr::walk(gh_md_files_structure, ~ expect_true(all(grepl("\\.md|\\.qmd|\\.Rmd", .)))) + test_mocker$cache(gh_md_files_structure) }) -test_that("get_files_structure_from_orgs pulls files structure for repositories in orgs", { - github_testhost_priv <- create_github_testhost( - repos = c("r-world-devs/GitStats", "openpharma/DataFakeR"), - mode = "private" - ) +test_that("get_files_structure_from_orgs", { mockery::stub( github_testhost_priv$get_files_structure_from_orgs, "graphql_engine$get_files_structure_from_org", - test_mocker$use("gh_files_structure") + test_mocker$use("gh_md_files_structure") ) - expect_snapshot( - gh_files_structure_from_orgs <- github_testhost_priv$get_files_structure_from_orgs( - pattern = NULL, - depth = 2L, - verbose = TRUE - ) + gh_files_structure_from_orgs <- github_testhost_priv$get_files_structure_from_orgs( + pattern = "\\.md|\\.qmd|\\.Rmd", + depth = Inf, + verbose = FALSE + ) + expect_gt( + length(gh_files_structure_from_orgs), + 0 ) expect_equal( names(gh_files_structure_from_orgs), - c("r-world-devs", "openpharma") + "test-org" ) - expect_true(any(grepl("\\.md|\\.qmd|\\.Rmd", gh_files_structure_from_orgs[[1]]))) test_mocker$cache(gh_files_structure_from_orgs) }) @@ -198,10 +219,9 @@ test_that("get_path_from_files_structure gets file path from files structure", { file_path <- test_graphql_github$get_path_from_files_structure( host_files_structure = test_mocker$use("gh_files_structure_from_orgs"), only_text_files = FALSE, - org = "r-world-devs", - repo = "GitStats" + org = "test-org", + repo = "TestRepo" ) - expect_equal(typeof(file_path), "character") expect_true(length(file_path) > 0) }) @@ -219,9 +239,9 @@ test_that("get_files_structure pulls files structure for repositories in orgs", ) expect_equal( names(gh_files_structure_from_orgs), - c("r-world-devs", "openpharma") + "test-org" ) - purrr::walk(gh_files_structure_from_orgs[[2]], function(repo_files) { + purrr::walk(gh_files_structure_from_orgs[[1]], function(repo_files) { expect_true(any(grepl("\\.md|\\.Rmd", repo_files))) }) test_mocker$cache(gh_files_structure_from_orgs) diff --git a/tests/testthat/test-get_files_structure-GitStats.R b/tests/testthat/test-get_files_structure-GitStats.R index 2b0da258..eff6fb16 100644 --- a/tests/testthat/test-get_files_structure-GitStats.R +++ b/tests/testthat/test-get_files_structure-GitStats.R @@ -5,13 +5,13 @@ test_that("get_files_structure_from_hosts works as expected", { test_mocker$use("gh_files_structure_from_orgs") ) files_structure_from_hosts <- test_gitstats_priv$get_files_structure_from_hosts( - pattern = "\\md", + pattern = "\\.md|\\.qmd\\.Rmd", depth = 1L, verbose = FALSE ) expect_equal(names(files_structure_from_hosts), c("github.com", "gitlab.com")) - expect_equal(names(files_structure_from_hosts[[1]]), c("r-world-devs", "openpharma")) + expect_equal(names(files_structure_from_hosts[[1]]), c("test-org")) files_structure_from_hosts[[2]] <- test_mocker$use("gl_files_structure_from_orgs") test_mocker$cache(files_structure_from_hosts) }) diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index 5f25aff3..1b2bd73d 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -33,12 +33,29 @@ test_that("`prepare_releases_table()` prepares releases table", { test_mocker$cache(releases_table) }) +test_that("`set_repositories` works", { + mockery::stub( + github_testhost_priv$set_repositories, + "private$get_all_repos", + test_mocker$use("gh_repos_table") + ) + repos_names <- github_testhost_priv$set_repositories() + expect_type(repos_names, "character") + expect_gt(length(repos_names), 0) + test_mocker$cache(repos_names) +}) + test_that("`get_release_logs()` pulls release logs in the table format", { mockery::stub( github_testhost$get_release_logs, "private$prepare_releases_table", test_mocker$use("releases_table") ) + mockery::stub( + github_testhost$get_release_logs, + "private$set_repositories", + test_mocker$use("repos_names") + ) releases_table <- github_testhost$get_release_logs( since = "2023-05-01", until = "2023-09-30", diff --git a/tests/testthat/test-get_urls_repos-GitHub.R b/tests/testthat/test-get_urls_repos-GitHub.R index f211f049..9cada53a 100644 --- a/tests/testthat/test-get_urls_repos-GitHub.R +++ b/tests/testthat/test-get_urls_repos-GitHub.R @@ -6,7 +6,7 @@ test_that("get_repos_urls() works", { ) gh_repos_urls <- test_rest_github$get_repos_urls( type = "web", - org = "r-world-devs" + org = "test-org" ) expect_gt( length(gh_repos_urls), @@ -16,20 +16,18 @@ test_that("get_repos_urls() works", { }) test_that("get_all_repos_urls prepares api repo_urls vector", { - github_testhost_priv <- create_github_testhost(orgs = c("r-world-devs", "openpharma"), + github_testhost_priv <- create_github_testhost(orgs = "test-org", mode = "private") mockery::stub( test_rest_github$get_repos_urls, "self$response", test_fixtures$github_repositories_rest_response ) - gh_api_repos_urls <- github_testhost_priv$get_all_repos_urls( - type = "api", - verbose = FALSE + gh_api_repos_urls <- test_rest_github$get_repos_urls( + type = "api" ) expect_gt(length(gh_api_repos_urls), 0) - expect_true(any(grepl("openpharma", gh_api_repos_urls))) - expect_true(any(grepl("r-world-devs", gh_api_repos_urls))) + expect_true(any(grepl("test-org", gh_api_repos_urls))) expect_true(all(grepl("api", gh_api_repos_urls))) test_mocker$cache(gh_api_repos_urls) }) @@ -38,15 +36,16 @@ test_that("get_all_repos_urls prepares web repo_urls vector", { mockery::stub( github_testhost_priv$get_all_repos_urls, "rest_engine$get_repos_urls", - test_mocker$use("gh_repos_urls") + test_mocker$use("gh_repos_urls"), + depth = 2L ) gh_repos_urls <- github_testhost_priv$get_all_repos_urls( type = "web", verbose = FALSE ) expect_gt(length(gh_repos_urls), 0) - expect_true(any(grepl("r-world-devs", gh_repos_urls))) - expect_true(all(grepl("https://github.com/", gh_repos_urls))) + expect_true(any(grepl("test-org", gh_repos_urls))) + expect_true(all(grepl("https://testhost.com/", gh_repos_urls))) test_mocker$cache(gh_repos_urls) }) @@ -59,7 +58,7 @@ test_that("get_repo_url_from_response retrieves repositories URLS", { expect_gt(length(gh_repo_api_urls), 0) test_mocker$cache(gh_repo_api_urls) gh_repo_web_urls <- github_testhost_priv$get_repo_url_from_response( - search_response = test_mocker$use("gh_search_response_in_file"), + search_response = test_mocker$use("gh_search_repos_response"), type = "web" ) expect_type(gh_repo_web_urls, "character") diff --git a/tests/testthat/test-get_urls_repos-GitStats.R b/tests/testthat/test-get_urls_repos-GitStats.R index 079a3f03..87266b6f 100644 --- a/tests/testthat/test-get_urls_repos-GitStats.R +++ b/tests/testthat/test-get_urls_repos-GitStats.R @@ -14,7 +14,7 @@ test_that("get_repos_urls_from_hosts gets data from the hosts", { expect_type(repos_urls_from_hosts, "character") expect_gt(length(repos_urls_from_hosts), 0) expect_true(any(grepl("gitlab.com/api", repos_urls_from_hosts))) - expect_true(any(grepl("api.github", repos_urls_from_hosts))) + expect_true(any(grepl("testhost.com/api", repos_urls_from_hosts))) test_mocker$cache(repos_urls_from_hosts) }) diff --git a/tests/testthat/test-get_user-GitHub.R b/tests/testthat/test-get_user-GitHub.R index 6638be43..250d6c3c 100644 --- a/tests/testthat/test-get_user-GitHub.R +++ b/tests/testthat/test-get_user-GitHub.R @@ -37,9 +37,9 @@ test_that("GitHost gets users tables", { "private$prepare_user_table", test_mocker$use("gh_user_table") ) - users_table <- github_testhost$get_users( + github_users <- github_testhost$get_users( users = c("testuser1", "testuser2") ) - expect_users_table(users_table) - test_mocker$cache(users_table) + expect_users_table(github_users) + test_mocker$cache(github_users) }) diff --git a/tests/testthat/test-get_user-GitLab.R b/tests/testthat/test-get_user-GitLab.R index 726b5bc7..4b7bed2f 100644 --- a/tests/testthat/test-get_user-GitLab.R +++ b/tests/testthat/test-get_user-GitLab.R @@ -37,10 +37,11 @@ test_that("get_users build users table for GitLab", { "private$prepare_user_table", test_mocker$use("gl_user_table") ) - users_result <- gitlab_testhost$get_users( + gitlab_users <- gitlab_testhost$get_users( users = c("testuser1", "testuser2") ) expect_users_table( - users_result + gitlab_users ) + test_mocker$cache(gitlab_users) }) diff --git a/tests/testthat/test-get_users-GitStats.R b/tests/testthat/test-get_users-GitStats.R index deca969a..230983eb 100644 --- a/tests/testthat/test-get_users-GitStats.R +++ b/tests/testthat/test-get_users-GitStats.R @@ -1,7 +1,23 @@ +test_that("get_users_from_hosts work", { + mockery::stub( + test_gitstats_priv$get_users_from_hosts, + "host$get_users", + test_mocker$use("github_users") + ) + users_from_hosts <- test_gitstats_priv$get_users_from_hosts() + expect_users_table(users_from_hosts) + test_mocker$cache(users_from_hosts) +}) + test_that("GitStats get users info", { test_gitstats <- create_test_gitstats(hosts = 2) + mockery::stub( + test_gitstats$get_users, + "private$get_users_from_hosts", + test_mocker$use("users_from_hosts") + ) users_result <- test_gitstats$get_users( - c("maciekbanas", "kalimu", "marcinkowskak"), + c("test_user1", "test_user2"), verbose = FALSE ) expect_users_table( diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 46ea3660..4240b22c 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -54,6 +54,12 @@ test_that("`check_token()` prints error when token exists but does not grant acc }) test_that("when token is proper token is passed", { + skip_on_cran() + github_testhost_priv <- create_github_testhost( + orgs = "test-org", + token = Sys.getenv("GITHUB_PAT"), + mode = "private" + ) expect_equal( github_testhost_priv$check_token(Sys.getenv("GITHUB_PAT")), Sys.getenv("GITHUB_PAT") @@ -61,6 +67,12 @@ test_that("when token is proper token is passed", { }) test_that("check_endpoint returns TRUE if they are correct", { + skip_on_cran() + github_testhost_priv <- create_github_testhost( + org = "test-org", + token = Sys.getenv("GITHUB_PAT"), + mode = "private" + ) expect_true( github_testhost_priv$check_endpoint( endpoint = "https://api.github.com/repos/r-world-devs/GitStats", @@ -75,6 +87,12 @@ test_that("check_endpoint returns TRUE if they are correct", { }) test_that("check_endpoint returns error if they are not correct", { + skip_on_cran() + github_testhost_priv <- create_github_testhost( + orgs = "test-org", + token = Sys.getenv("GITHUB_PAT"), + mode = "private" + ) expect_snapshot_error( check <- github_testhost_priv$check_endpoint( endpoint = "https://api.github.com/repos/r-worlddevs/GitStats", diff --git a/tests/testthat/test-show_orgs.R b/tests/testthat/test-show_orgs.R index 22320700..6a4120a0 100644 --- a/tests/testthat/test-show_orgs.R +++ b/tests/testthat/test-show_orgs.R @@ -1,6 +1,6 @@ test_that("show_orgs() returns orgs", { expect_equal( show_orgs(test_gitstats), - c("r-world-devs", "mbtests") + c("github_test_org", "gitlab_test_group") ) }) diff --git a/tests/testthat/test-z-GitStats.R b/tests/testthat/test-z-GitStats.R index 10439310..1782bbe5 100644 --- a/tests/testthat/test-z-GitStats.R +++ b/tests/testthat/test-z-GitStats.R @@ -88,7 +88,7 @@ test_that("show_orgs print orgs properly", { test_gitstats <- create_test_gitstats(hosts = 2) expect_equal( test_gitstats$show_orgs(), - c("r-world-devs", "mbtests") + c("github_test_org", "gitlab_test_group") ) }) From 27309464dbb02fa141d2d8ac21c0338d5fb8ed8a Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 7 Oct 2024 10:42:47 +0000 Subject: [PATCH 07/94] Fix function. --- R/EngineGraphQLGitHub.R | 2 ++ tests/testthat/test-get_files_content-GitHub.R | 1 + 2 files changed, 3 insertions(+) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 80f8eb93..f8dd3831 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -102,6 +102,7 @@ EngineGraphQLGitHub <- R6::R6Class( org_files_list <- private$get_repositories_with_files( repositories = repositories, def_branches = def_branches, + org = org, file_paths = file_paths, host_files_structure = host_files_structure, only_text_files = only_text_files, @@ -240,6 +241,7 @@ EngineGraphQLGitHub <- R6::R6Class( get_repositories_with_files = function(repositories, def_branches, + org, host_files_structure, only_text_files, file_paths, diff --git a/tests/testthat/test-get_files_content-GitHub.R b/tests/testthat/test-get_files_content-GitHub.R index 642e5235..4d3a9423 100644 --- a/tests/testthat/test-get_files_content-GitHub.R +++ b/tests/testthat/test-get_files_content-GitHub.R @@ -73,6 +73,7 @@ test_that("get_repositories_with_files works", { gh_repositories_with_files <- test_graphql_github_priv$get_repositories_with_files( repositories = c("test_repo_1", "test_repo_2", "test_repo_3", "test_repo_4", "test_repo_5"), def_branches = c("test_branch_1", "test_branch_2", "test_branch_3", "test_branch_4", "test_branch_5"), + org = "test-org", file_paths = "test_files", only_text_files = TRUE, host_files_structure = NULL, From c9fd7a51ccb71bc5664041a5e05837cb6211476e Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 7 Oct 2024 13:30:32 +0000 Subject: [PATCH 08/94] Mock files structure tests for GitLab. --- .../_snaps/get_files_structure-GitLab.md | 4 +- tests/testthat/helper-fixtures.R | 3 + .../test-get_files_structure-GitLab.R | 73 ++++++++++--------- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/tests/testthat/_snaps/get_files_structure-GitLab.md b/tests/testthat/_snaps/get_files_structure-GitLab.md index a4c86492..5fdb0493 100644 --- a/tests/testthat/_snaps/get_files_structure-GitLab.md +++ b/tests/testthat/_snaps/get_files_structure-GitLab.md @@ -9,10 +9,10 @@ Code gl_files_structure_from_orgs <- gitlab_testhost_priv$ - get_files_structure_from_orgs(pattern = "\\.md|\\.R", depth = 1L, verbose = TRUE, + get_files_structure_from_orgs(pattern = "\\.md", depth = 1L, verbose = TRUE, progress = FALSE) Message - i [Host:GitLab][Engine:GraphQl][Scope:mbtests] Pulling files structure...[files matching pattern: '\.md|\.R']... + i [Host:GitLab][Engine:GraphQl][Scope:mbtests] Pulling files structure...[files matching pattern: '\.md']... # get_files_content makes use of files_structure diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index f9be55e5..eb6618b2 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -467,6 +467,9 @@ test_fixtures$gitlab_files_tree_response <- list( ), list( "name" = "project_metadata.yaml" + ), + list( + "name" = "report.md" ) ) ) diff --git a/tests/testthat/test-get_files_structure-GitLab.R b/tests/testthat/test-get_files_structure-GitLab.R index 9c5bf19d..5898ec98 100644 --- a/tests/testthat/test-get_files_structure-GitLab.R +++ b/tests/testthat/test-get_files_structure-GitLab.R @@ -70,7 +70,7 @@ test_that("get_files_structure_from_repo() pulls files structure from repo", { test_that("only files with certain pattern are retrieved", { md_files_structure <- test_graphql_gitlab_priv$filter_files_by_pattern( files_structure = test_mocker$use("gl_files_structure"), - pattern = "\\.md|\\.qmd|\\.Rmd" + pattern = "\\.md" ) files_structure <- test_mocker$use("files_structure") expect_true( @@ -110,6 +110,14 @@ test_that("get_files_structure_from_repo() pulls files structure (files matching }) test_that("GitLab GraphQL Engine pulls files structure from repositories", { + gl_repos_data <- list( + "repositories" = c("tests_repo_1", "tests_repo_2") + ) + mockery::stub( + test_graphql_gitlab$get_files_structure_from_org, + "private$get_repos_data", + gl_repos_data + ) mockery::stub( test_graphql_gitlab$get_files_structure_from_org, "private$get_files_structure_from_repo", @@ -117,23 +125,36 @@ test_that("GitLab GraphQL Engine pulls files structure from repositories", { ) gl_files_structure <- test_graphql_gitlab$get_files_structure_from_org( org = "mbtests", - repos = c("graphql_tests"), + repos = c("tests_repo_1", "tests_repo_2"), verbose = FALSE, progress = FALSE ) purrr::walk(gl_files_structure, ~ expect_true(length(.) > 0)) expect_equal( names(gl_files_structure), - c("graphql_tests") + c("tests_repo_1", "tests_repo_2") ) purrr::walk(gl_files_structure, ~ expect_false(all(grepl("/$", .)))) # no empty dirs - test_mocker$cache(gl_files_structure) }) test_that("GitLab GraphQL Engine pulls files structure from repositories", { + gl_repos_data <- list( + "repositories" = c("tests_repo_1", "tests_repo_2") + ) + mockery::stub( + test_graphql_gitlab$get_files_structure_from_org, + "private$get_repos_data", + gl_repos_data + ) + mockery::stub( + test_graphql_gitlab$get_files_structure_from_org, + "private$get_files_structure_from_repo", + test_mocker$use("gl_md_files_structure") + ) gl_files_structure_shallow <- test_graphql_gitlab$get_files_structure_from_org( org = "mbtests", - repos = c("gitstatstesting", "graphql_tests"), + repos = c("tests_repo_1", "tests_repo_2"), + pattern = "\\.md", depth = 1L, verbose = FALSE, progress = FALSE @@ -141,19 +162,21 @@ test_that("GitLab GraphQL Engine pulls files structure from repositories", { purrr::walk(gl_files_structure_shallow, ~ expect_true(length(.) > 0)) expect_equal( names(gl_files_structure_shallow), - c("graphql_tests", "gitstatstesting") + c("tests_repo_1", "tests_repo_2") ) purrr::walk(gl_files_structure_shallow, ~ expect_false(all(grepl("/", .)))) # no dirs + test_mocker$cache(gl_files_structure_shallow) }) test_that("get_files_structure_from_orgs pulls files structure for repositories in orgs", { - gitlab_testhost_priv <- create_gitlab_testhost( - repos = c("mbtests/gitstatstesting", "mbtests/graphql_tests"), - mode = "private" + mockery::stub( + gitlab_testhost_priv$get_files_structure_from_orgs, + "graphql_engine$get_files_structure_from_org", + test_mocker$use("gl_files_structure_shallow") ) expect_snapshot( gl_files_structure_from_orgs <- gitlab_testhost_priv$get_files_structure_from_orgs( - pattern = "\\.md|\\.R", + pattern = "\\.md", depth = 1L, verbose = TRUE, progress = FALSE @@ -164,27 +187,7 @@ test_that("get_files_structure_from_orgs pulls files structure for repositories c("mbtests") ) purrr::walk(gl_files_structure_from_orgs[[1]], function(repo_files) { - expect_true(all(grepl("\\.md|\\.R", repo_files))) - }) -}) - -test_that("get_files_structure_from_orgs pulls files structure for all repositories in orgs", { - gitlab_testhost_priv <- create_gitlab_testhost( - orgs = c("mbtests", "mbtestapps"), - mode = "private" - ) - gl_files_structure_from_orgs <- gitlab_testhost_priv$get_files_structure_from_orgs( - pattern = "\\.md|\\.R", - depth = 1L, - verbose = FALSE, - progress = FALSE - ) - expect_equal( - names(gl_files_structure_from_orgs), - c("mbtests", "mbtestapps") - ) - purrr::walk(gl_files_structure_from_orgs[[1]], function(repo_files) { - expect_true(all(grepl("\\.md|\\.R", repo_files))) + expect_true(all(grepl("\\.md", repo_files))) }) test_mocker$cache(gl_files_structure_from_orgs) }) @@ -211,17 +214,17 @@ test_that("get_files_structure pulls files structure for repositories in orgs", test_mocker$use("gl_files_structure_from_orgs") ) gl_files_structure_from_orgs <- gitlab_testhost$get_files_structure( - pattern = "\\.md|\\.R", - depth = 2L, + pattern = "\\.md", + depth = 1L, verbose = FALSE, progress = FALSE ) expect_equal( names(gl_files_structure_from_orgs), - c("mbtests", "mbtestapps") + c("mbtests") ) purrr::walk(gl_files_structure_from_orgs[[1]], function(repo_files) { - expect_true(all(grepl("\\.md|\\.R", repo_files))) + expect_true(all(grepl("\\.md", repo_files))) }) test_mocker$cache(gl_files_structure_from_orgs) }) From 057c64be19021dcb67ec96cf52e2d4223e2ef22d Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 7 Oct 2024 13:52:56 +0000 Subject: [PATCH 09/94] Mock some responses. --- R/EngineRestGitLab.R | 7 +++--- tests/testthat/helper-fixtures.R | 15 ++++++++++++ tests/testthat/test-01-get_repos-GitLab.R | 28 ++++++++++++++++------- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 86bf8209..6820a147 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -59,7 +59,7 @@ EngineRestGitLab <- R6::R6Class( private$map_search_into_repos( progress = progress ) %>% - private$pull_repos_languages( + private$get_repos_languages( progress = progress ) } @@ -87,7 +87,6 @@ EngineRestGitLab <- R6::R6Class( issues <- purrr::map(repos_table$repo_id, function(repos_id) { id <- gsub("gid://gitlab/Project/", "", repos_id) issues_endpoint <- paste0(self$rest_api_url, "/projects/", id, "/issues_statistics") - self$response( endpoint = issues_endpoint )[["statistics"]][["counts"]] @@ -237,7 +236,7 @@ EngineRestGitLab <- R6::R6Class( endpoint = repo_endpoint ) full_repos_list <- repos_response %>% - private$pull_repos_languages( + private$get_repos_languages( progress = progress ) return(full_repos_list) @@ -301,7 +300,7 @@ EngineRestGitLab <- R6::R6Class( }, # Pull languages of repositories. - pull_repos_languages = function(repos_list, progress) { + get_repos_languages = function(repos_list, progress) { repos_list_with_languages <- purrr::map(repos_list, function(repo) { id <- repo$id repo$languages <- names(self$response(paste0(private$endpoints[["projects"]], id, "/languages"))) diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index eb6618b2..2c6d59e1 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -735,3 +735,18 @@ test_fixtures$github_issues_response <- list( test_fixtures$github_open_issue_response, test_fixtures$github_closed_issue_response ) + +test_fixtures$gitlab_issues_response <- list( + "statistics" = list( + "counts" = list( + "all" = 3, + "closed" = 2, + "opened" = 1 + ) + ) +) + +test_fixtures$gitlab_languages_response <- list( + "Python" = 100, + "R" = 50 +) diff --git a/tests/testthat/test-01-get_repos-GitLab.R b/tests/testthat/test-01-get_repos-GitLab.R index 2d7438a4..67846f96 100644 --- a/tests/testthat/test-01-get_repos-GitLab.R +++ b/tests/testthat/test-01-get_repos-GitLab.R @@ -90,16 +90,20 @@ test_that("`map_search_into_repos()` works", { test_mocker$cache(gl_search_repos_by_code) }) -test_that("`pull_repos_languages` works", { +test_that("`get_repos_languages` works", { repos_list <- test_mocker$use("gl_search_repos_by_code") repos_list[[1]]$id <- "45300912" - suppressMessages( - repos_list_with_languages <- test_rest_gitlab_priv$pull_repos_languages( - repos_list = repos_list, - progress = FALSE - ) + mockery::stub( + test_rest_gitlab_priv$get_repos_languages, + "self$response", + test_fixtures$gitlab_languages_response + ) + repos_list_with_languages <- test_rest_gitlab_priv$get_repos_languages( + repos_list = repos_list, + progress = FALSE ) purrr::walk(repos_list_with_languages, ~ expect_list_contains(., "languages")) + expect_equal(repos_list_with_languages[[1]]$languages, c("Python", "R")) }) test_that("`prepare_repos_table()` prepares repos table", { @@ -121,10 +125,8 @@ test_that("GitHost adds `repo_api_url` column to GitLab repos table", { test_that("`tailor_repos_response()` tailors precisely `repos_list`", { gl_repos_by_code <- test_mocker$use("gl_search_repos_by_code") - gl_repos_by_code_tailored <- gitlab_testhost_priv$tailor_repos_response(gl_repos_by_code) - gl_repos_by_code_tailored %>% expect_type("list") %>% expect_length(length(gl_repos_by_code)) @@ -158,6 +160,11 @@ test_that("GitHost prepares table from GitLab repositories response", { }) test_that("`get_repos_issues()` adds issues to repos table", { + mockery::stub( + test_rest_gitlab$get_repos_issues, + "self$response", + test_fixtures$gitlab_issues_response + ) gl_repos_by_code_table <- test_mocker$use("gl_repos_by_code_table") gl_repos_by_code_table <- test_rest_gitlab$get_repos_issues( gl_repos_by_code_table, @@ -175,6 +182,11 @@ test_that("`get_repos_issues()` adds issues to repos table", { }) test_that("`get_repos_contributors()` adds contributors to repos table", { + mockery::stub( + test_rest_gitlab$get_repos_contributors, + "private$get_contributors_from_repo", + "Maciej Banas" + ) gl_repos_table_with_contributors <- test_rest_gitlab$get_repos_contributors( test_mocker$use("gl_repos_table_with_api_url"), progress = FALSE From 0822050c36dc4ff700cfe2e7dd5fba8ad95259d3 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 8 Oct 2024 14:46:09 +0000 Subject: [PATCH 10/94] Fix error by getting back from `|>` to `%>%` operator and refactor for loops to purrr iterations. The `|>` does not support `.` argument in pipelines. --- R/EngineRestGitHub.R | 65 +++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index eccd0ca2..694a688b 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -167,16 +167,24 @@ EngineRestGitHub <- R6::R6Class( total_n, byte_max = "384000") { if (total_n >= 0 & total_n < 1e3) { - resp_list <- list() - for (page in 1:(total_n %/% 100)) { - resp_list <- self$response( - paste0(search_endpoint, "+size:0..", byte_max, "&page=", page, "&per_page=100") - )[["items"]] %>% - append(resp_list, .) - } - resp_list + page_iterator <- 1:(total_n %/% 100) + items_list <- purrr::map(page_iterator, function(page) { + response <- self$response( + paste0( + search_endpoint, + "+size:0..", + byte_max, + "&page=", + page, + "&per_page=100" + ) + ) + return(response[["items"]]) + }) %>% + purrr::list_flatten() + return(items_list) } else if (total_n >= 1e3) { - resp_list <- list() + items_list <- list() index <- c(0, 50) spinner <- cli::make_spinner( which = "timeTravel", @@ -187,28 +195,29 @@ EngineRestGitHub <- R6::R6Class( while (index[2] < as.numeric(byte_max)) { size_formula <- paste0("+size:", as.character(index[1]), "..", as.character(index[2])) spinner$spin() - n_count <- tryCatch( - { - self$response(paste0(search_endpoint, size_formula))[["total_count"]] - }, - error = function(e) { - NULL - } - ) + n_count <- tryCatch({ + self$response(paste0(search_endpoint, size_formula))[["total_count"]] + }, error = function(e) { + NULL + }) if (is.null(n_count)) { NULL } else if ((n_count - 1) %/% 100 > 0) { - for (page in (1:(n_count %/% 100) + 1)) { - resp_list <- self$response(paste0(search_endpoint, - size_formula, - "&page=", - page, - "&per_page=100"))[["items"]] |> - append(resp_list, .) - } + total_pages <- 1:(n_count %/% 100) + 1 + items_part_list <- purrr::map(total_pages, function(page) { + self$response(paste0(search_endpoint, + size_formula, + "&page=", + page, + "&per_page=100"))[["items"]] + }) %>% + purrr::list_flatten() + items_list <- append(items_list, items_part_list) } else if ((n_count - 1) %/% 100 == 0) { - resp_list <- self$response(paste0(search_endpoint, size_formula, "&page=1&per_page=100"))[["items"]] |> - append(resp_list, .) + response <- self$response(paste0(search_endpoint, + size_formula, + "&page=1&per_page=100")) + items_list <- append(items_list, response[["items"]]) } index[1] <- index[2] if (index[2] < 1e3) { @@ -225,7 +234,7 @@ EngineRestGitHub <- R6::R6Class( } } spinner$finish() - resp_list + return(items_list) } }, From bd6149fda5d0b1d0dfe8efdf04982feb4486ae8c Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 8 Oct 2024 15:08:29 +0000 Subject: [PATCH 11/94] Add test with large search response. --- tests/testthat/helper-fixtures.R | 68 +++++++++-------------- tests/testthat/test-01-get_repos-GitHub.R | 20 ++++++- 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 2c6d59e1..53de5a07 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -393,52 +393,34 @@ test_fixtures$gitlab_search_response <- list( ) ) +github_search_item <- list( + "name" = "test1.R", + "path" = "examples/test1.R", + "sha" = "0d42b9d23ddfc0bca1", + "url" = "test_url", + "git_url" = "test_git_url", + "html_url" = "test_html_url", + "repository" = list( + "id" = 627452680, + "url" = "https://api.github.com/repos/r-world-devs/GitStats", + "html_url" = "https://github.com/r-world-devs/GitStats" + ), + "score" = 1 +) + test_fixtures$github_search_response <- list( - "total_count" = 3, + "total_count" = 250, "incomplete_results" = FALSE, "items" = list( - list( - "name" = "test1.R", - "path" = "examples/test1.R", - "sha" = "0d42b9d23ddfc0bca1", - "url" = "test_url", - "git_url" = "test_git_url", - "html_url" = "test_html_url", - "repository" = list( - "id" = 627452680, - "url" = "https://api.github.com/repos/r-world-devs/GitStats", - "html_url" = "https://github.com/r-world-devs/GitStats" - ), - "score" = 1 - ), - list( - "name" = "test2.R", - "path" = "tests/test2.R", - "sha" = "01238xb", - "url" = "test_url", - "git_url" = "test_git_url", - "html_url" = "test_html_url", - "repository" = list( - "id" = 604718884, - "url" = "https://api.github.com/repos/r-world-devs/GitStats", - "html_url" = "https://github.com/r-world-devs/GitStats" - ), - "score" = 1 - ), - list( - "name" = "test3.R", - "path" = "R/test3.R", - "sha" = "20e19af2dda26d04f6", - "url" = "test_url", - "git_url" = "test_git_url", - "html_url" = "test_html_url", - "repository" = list( - "id" = 495151911, - "url" = "https://api.github.com/repos/r-world-devs/GitStats", - "html_url" = "https://github.com/r-world-devs/GitStats" - ), - "score" = 1 - ) + rep(github_search_item, 250) + ) +) + +test_fixtures$github_search_response_large <- list( + "total_count" = 1001, + "incomplete_results" = FALSE, + "items" = list( + rep(github_search_item, 1001) ) ) diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 4dc39f81..50437b1d 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -44,7 +44,7 @@ test_that("`get_repos_from_org()` prepares formatted list", { # REST Engine search repos by code -test_that("`search_response()` performs search with limit under 100", { +test_that("`search_response()` performs search with limit under 1000", { total_n <- test_fixtures$github_search_response[["total_count"]] mockery::stub( test_rest_github_priv$search_response, @@ -53,8 +53,22 @@ test_that("`search_response()` performs search with limit under 100", { ) gh_search_repos_response <- test_rest_github_priv$search_response( search_endpoint = test_mocker$use("search_endpoint"), - total_n = total_n, - byte_max = 384000 + total_n = total_n + ) + expect_gh_search_response(gh_search_repos_response) + test_mocker$cache(gh_search_repos_response) +}) + +test_that("`search_response()` performs search with limit over 1000", { + total_n <- test_fixtures$github_search_response_large[["total_count"]] + mockery::stub( + test_rest_github_priv$search_response, + "self$response", + test_fixtures$github_search_response_large + ) + gh_search_repos_response <- test_rest_github_priv$search_response( + search_endpoint = test_mocker$use("search_endpoint"), + total_n = total_n ) expect_gh_search_response(gh_search_repos_response) test_mocker$cache(gh_search_repos_response) From b3e8be2eba8040f75423ddfd3750fff7a2989e57 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 9 Oct 2024 07:52:37 +0000 Subject: [PATCH 12/94] Fix when r_package_usage output is empty. --- R/GitStats.R | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/R/GitStats.R b/R/GitStats.R index 39058e4a..96071593 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -479,12 +479,23 @@ GitStats <- R6::R6Class( package_name = package_name, only_loading = only_loading, verbose = verbose - ) %>% - private$set_object_class( + ) + if (nrow(R_package_usage) > 0) { + R_package_usage <- private$set_object_class( + object = R_package_usage, class = "R_package_usage", attr_list = args_list ) - private$save_to_storage(R_package_usage) + private$save_to_storage(R_package_usage) + } else { + if (verbose) { + cli::cli_alert_warning( + cli::col_yellow( + "No usage of R packages found." + ) + ) + } + } } else { R_package_usage <- private$get_from_storage( table = "R_package_usage", @@ -956,13 +967,15 @@ GitStats <- R6::R6Class( repos_using_package ) ) - duplicated_repos <- package_usage_table$api_url[duplicated(package_usage_table$api_url)] - package_usage_table <- package_usage_table[!duplicated(package_usage_table$api_url), ] - package_usage_table <- package_usage_table %>% - dplyr::mutate( - package_usage = ifelse(api_url %in% duplicated_repos, "import, library", package_usage) - ) - rownames(package_usage_table) <- seq_len(nrow(package_usage_table)) + if (nrow(package_usage_table) > 0) { + duplicated_repos <- package_usage_table$api_url[duplicated(package_usage_table$api_url)] + package_usage_table <- package_usage_table[!duplicated(package_usage_table$api_url), ] + package_usage_table <- package_usage_table %>% + dplyr::mutate( + package_usage = ifelse(api_url %in% duplicated_repos, "import, library", package_usage) + ) + rownames(package_usage_table) <- seq_len(nrow(package_usage_table)) + } return(package_usage_table) }, @@ -1005,12 +1018,12 @@ GitStats <- R6::R6Class( verbose = FALSE, progress = FALSE ) - if (nrow(repos_with_package) > 0) { + if (!is.null(repos_with_package)) { repos_with_package <- repos_with_package[!duplicated(repos_with_package$api_url), ] repos_with_package$package_usage <- "import" + repos_with_package <- repos_with_package %>% + dplyr::select(repo_name, organization, fullname, platform, repo_url, api_url, package_usage) } - repos_with_package <- repos_with_package %>% - dplyr::select(repo_name, organization, fullname, platform, repo_url, api_url, package_usage) return(repos_with_package) }, From 30b79973ecd46773148b0d1105916f45ceade721 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 9 Oct 2024 07:58:29 +0000 Subject: [PATCH 13/94] Limit expectation to one object. --- tests/testthat/helper-expect-responses.R | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/testthat/helper-expect-responses.R b/tests/testthat/helper-expect-responses.R index 1291fecc..798275e3 100644 --- a/tests/testthat/helper-expect-responses.R +++ b/tests/testthat/helper-expect-responses.R @@ -16,12 +16,10 @@ expect_gh_search_response <- function(object) { object, "list" ) - purrr::walk(object, ~ { - expect_list_contains( - ., - c("name", "path", "sha", "url", "git_url", "html_url", "repository", "score") - ) - }) + expect_list_contains( + object[[1]], + c("name", "path", "sha", "url", "git_url", "html_url", "repository", "score") + ) } expect_gl_repos_rest_response <- function(object) { From 6123610f1317c56a145d7abf60133e77068b26cb Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 9 Oct 2024 07:58:47 +0000 Subject: [PATCH 14/94] Add test when empty response. --- tests/testthat/test-get_usage_R_package.R | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/testthat/test-get_usage_R_package.R b/tests/testthat/test-get_usage_R_package.R index 0e8ac535..025c923f 100644 --- a/tests/testthat/test-get_usage_R_package.R +++ b/tests/testthat/test-get_usage_R_package.R @@ -37,3 +37,17 @@ test_that("get_R_package_usage_from_hosts works as expected", { expect_package_usage_table(R_package_usage_table) test_mocker$cache(R_package_usage_table) }) + +test_that("when get_R_package_usage_from_hosts output is empty return warning", { + test_gitstats <- create_test_gitstats(hosts = 2) + mockery::stub( + test_gitstats$get_R_package_usage, + "private$get_R_package_usage_from_hosts", + data.frame() + ) + expect_snapshot( + test_gitstats$get_R_package_usage( + package_name = "shiny", only_loading = FALSE, verbose = TRUE + ) + ) +}) From ef7602514b008bb05fc0019a6f16d324d6f268a5 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 9 Oct 2024 08:17:17 +0000 Subject: [PATCH 15/94] New snap. --- tests/testthat/_snaps/get_usage_R_package.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/testthat/_snaps/get_usage_R_package.md diff --git a/tests/testthat/_snaps/get_usage_R_package.md b/tests/testthat/_snaps/get_usage_R_package.md new file mode 100644 index 00000000..69eef51f --- /dev/null +++ b/tests/testthat/_snaps/get_usage_R_package.md @@ -0,0 +1,10 @@ +# when get_R_package_usage_from_hosts output is empty return warning + + Code + test_gitstats$get_R_package_usage(package_name = "shiny", only_loading = FALSE, + verbose = TRUE) + Message + ! No usage of R packages found. + Output + data frame with 0 columns and 0 rows + From ce37af8ef4c454cdf35d97112ee1b92376af37b3 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 9 Oct 2024 08:21:15 +0000 Subject: [PATCH 16/94] Bump version, add NEWS. --- DESCRIPTION | 2 +- NEWS.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index bca8688e..130709c4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.0 +Version: 2.1.0.9000 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), diff --git a/NEWS.md b/NEWS.md index be7a48fa..24fc24b9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# GitStats (development version) + +- Fixed getting large search responses for GitHub ([#491](https://github.com/r-world-devs/GitStats/issues/491)). + # GitStats 2.1.0 ## New features: From d8cc7f7222afc896639c06419d7fc4b4e7e46ddb Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 9 Oct 2024 14:55:11 +0000 Subject: [PATCH 17/94] Add getting GitHub repositories for individual user. Together with new feature a new method was introduced to check if owner is a user or an organization. --- R/EngineGraphQLGitHub.R | 28 +++++++++++++------ R/GQLQueryGitHub.R | 59 +++++++++++++++++++++++++++-------------- R/GitHost.R | 27 +++++++++++++++++-- 3 files changed, 84 insertions(+), 30 deletions(-) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index f8dd3831..80ba97a3 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -41,16 +41,22 @@ EngineGraphQLGitHub <- R6::R6Class( }, # Pull all repositories from organization - get_repos_from_org = function(org = NULL) { + get_repos_from_org = function(org = NULL, + type = c("organization", "user")) { full_repos_list <- list() next_page <- TRUE repo_cursor <- "" while (next_page) { repos_response <- private$get_repos_page( - org = org, + login = org, + type = type, repo_cursor = repo_cursor ) - repositories <- repos_response$data$repositoryOwner$repositories + repositories <- if (type == "organization") { + repos_response$data$repositoryOwner$repositories + } else { + repos_response$data$user$repositories + } repos_list <- repositories$nodes next_page <- repositories$pageInfo$hasNextPage if (is.null(next_page)) next_page <- FALSE @@ -162,14 +168,20 @@ EngineGraphQLGitHub <- R6::R6Class( private = list( # Wrapper over building GraphQL query and response. - get_repos_page = function(org = NULL, + get_repos_page = function(login = NULL, + type, repo_cursor = "") { - repos_query <- self$gql_query$repos_by_org( - repo_cursor = repo_cursor - ) + repos_query <- if (type == "organization") { + self$gql_query$repos_by_org() + } else { + self$gql_query$repos_by_user() + } response <- self$gql_response( gql_query = repos_query, - vars = list("org" = org) + vars = list( + "login" = login, + "repoCursor" = repo_cursor + ) ) return(response) }, diff --git a/R/GQLQueryGitHub.R b/R/GQLQueryGitHub.R index 38548b86..555d3269 100644 --- a/R/GQLQueryGitHub.R +++ b/R/GQLQueryGitHub.R @@ -34,32 +34,49 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub", ) }, + #' @description Query to check if a given string is user or organization. + user_or_org_query = + ' + query ($login: String!) { + user(login: $login) { + __typename + login + } + organization(login: $login) { + __typename + login + } + }' + , + #' @description Prepare query to get repositories from GitHub. - #' @param org An organization of repositories. - #' @param repo_cursor An end cursor for repositories page. #' @return A query. - repos_by_org = function(repo_cursor = "") { - if (nchar(repo_cursor) == 0) { - after_cursor <- repo_cursor - } else { - after_cursor <- paste0('after: "', repo_cursor, '" ') - } + repos_by_org = function() { paste0(' - query GetReposByOrg($org: String!) { - repositoryOwner(login: $org) { + query GetReposByOrg($login: String! $repoCursor: String!) { + repositoryOwner(login: $login) { ... on Organization { - repositories(first: 100 ', after_cursor, ') { - ', private$repository_field(), ' - } + ', private$repositories_field(), ' } } }') }, + #' @description Prepare query to get repositories from GitHub. + #' @return A query. + repos_by_user = function() { + paste0(' + query GetUsersRepos($login: String! $repoCursor: String!){ + user(login: $login) { + ', private$repositories_field(), ' + } + }' + ) + }, + #' @description Prepare query to get info on a GitHub user. - #' @param login A login of a user. #' @return A query. - user = function(login) { + user = function() { paste0(' query GetUser($user: String!) { user(login: $user) { @@ -212,9 +229,10 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub", }, # @description Helper to prepare repository query. - repository_field = function() { - paste0( - 'totalCount + repositories_field = function() { + ' + repositories(first: 100 after: $repoCursor) { + totalCount pageInfo { endCursor hasNextPage @@ -240,8 +258,9 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub", login } repo_url: url - }' - ) + } + } + ' } ) ) diff --git a/R/GitHost.R b/R/GitHost.R index 3b01b5fb..e390d3a1 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -412,13 +412,34 @@ GitHost <- R6::R6Class( ) private$repos_fullnames <- repos orgs_repos <- private$extract_repos_and_orgs(repos) - private$orgs <- names(orgs_repos) + private$orgs <- private$set_owner_type( + owners = names(orgs_repos) + ) private$repos <- unname(unlist(orgs_repos)) private$orgs_repos <- orgs_repos } } }, + # Set owner type + set_owner_type = function(owners) { + graphql_engine <- private$engines$graphql + user_or_org_query <- graphql_engine$gql_query$user_or_org_query + login_types <- purrr::map(owners, function(owner) { + response <- graphql_engine$gql_response( + gql_query = user_or_org_query, + vars = list( + "login" = owner + ) + ) + type <- purrr::discard(response$data, is.null) %>% + names() + attr(owner, "type") <- type + return(owner) + }) + return(login_types) + }, + # Check if repositories exist check_repositories = function(repos, verbose) { if (verbose) { @@ -593,6 +614,7 @@ GitHost <- R6::R6Class( } graphql_engine <- private$engines$graphql repos_table <- purrr::map(private$orgs, function(org) { + type <- attr(org, "type") org <- utils::URLdecode(org) if (!private$scan_all && verbose) { show_message( @@ -604,7 +626,8 @@ GitHost <- R6::R6Class( } repos <- private$set_repos(org) repos_table <- graphql_engine$get_repos_from_org( - org = org + org = org, + type = type ) %>% private$prepare_repos_table_from_graphql() if (!is.null(repos)) { From eb965dd051b635e8e9361cb09b9b810ca2ffebf2 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 9 Oct 2024 14:56:05 +0000 Subject: [PATCH 18/94] Bump version. --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 130709c4..27e860a9 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.0.9000 +Version: 2.1.0.9001 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), From 6477c7efc6ba68d03c75f7b7b697e4ad723c61f1 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 10 Oct 2024 07:27:32 +0000 Subject: [PATCH 19/94] Prepare test for helper setting type attributes. --- R/test_helpers.R | 12 ++++++++++-- tests/testthat/helper-fixtures.R | 20 ++++++++++++++++++++ tests/testthat/test-GitHost-helpers.R | 23 +++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 tests/testthat/test-GitHost-helpers.R diff --git a/R/test_helpers.R b/R/test_helpers.R index e582c54e..75a8e38d 100644 --- a/R/test_helpers.R +++ b/R/test_helpers.R @@ -36,12 +36,16 @@ GitHostGitHubTest <- R6::R6Class( private$check_if_public(host) private$token <- token private$set_graphql_url() - private$set_orgs_and_repos(orgs, repos, verbose = FALSE) + private$set_orgs_and_repos_mocked(orgs, repos) private$setup_test_engines() private$set_searching_scope(orgs, repos, verbose = FALSE) } ), private = list( + set_orgs_and_repos_mocked = function(orgs, repos) { + private$orgs <- orgs + private$repos <- repos + }, setup_test_engines = function() { private$engines$rest <- TestEngineRestGitHub$new( token = private$token, @@ -72,12 +76,16 @@ GitHostGitLabTest <- R6::R6Class( private$check_if_public(host) private$token <- token private$set_graphql_url() - private$set_orgs_and_repos(orgs, repos, verbose = FALSE) + private$set_orgs_and_repos_mocked(orgs, repos) private$setup_test_engines() private$set_searching_scope(orgs, repos, verbose = FALSE) } ), private = list( + set_orgs_and_repos_mocked = function(orgs, repos) { + private$orgs <- orgs + private$repos <- repos + }, setup_test_engines = function() { private$engines$rest <- TestEngineRestGitLab$new( token = private$token, diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 53de5a07..9e673f50 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -732,3 +732,23 @@ test_fixtures$gitlab_languages_response <- list( "Python" = 100, "R" = 50 ) + +test_fixtures$github_user_login <- list( + "data" = list( + "user" = list( + "__typename" = "User", + "login" = "test_user" + ), + "organization" = NULL + ) +) + +test_fixtures$github_org_login <- list( + "data" = list( + "user" = NULL, + "organization" = list( + "__typename" = "Organization", + "login" = "test_org" + ) + ) +) diff --git a/tests/testthat/test-GitHost-helpers.R b/tests/testthat/test-GitHost-helpers.R new file mode 100644 index 00000000..ade90edd --- /dev/null +++ b/tests/testthat/test-GitHost-helpers.R @@ -0,0 +1,23 @@ +test_that("set_owner_types sets attributes to owners list", { + mockery::stub( + github_testhost_priv$set_owner_type, + "graphql_engine$gql_response", + test_fixtures$github_user_login + ) + owner <- github_testhost_priv$set_owner_type( + owners = c("test_user") + ) + expect_equal(attr(owner[[1]], "type"), "user") + expect_equal(owner[[1]], "test_user", ignore_attr = TRUE) + + mockery::stub( + github_testhost_priv$set_owner_type, + "graphql_engine$gql_response", + test_fixtures$github_org_login + ) + owner <- github_testhost_priv$set_owner_type( + owners = c("test_org") + ) + expect_equal(attr(owner[[1]], "type"), "organization") + expect_equal(owner[[1]], "test_org", ignore_attr = TRUE) +}) From c39069e19b112551ff69b14a62ce55463d659dc7 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 10 Oct 2024 09:51:32 +0000 Subject: [PATCH 20/94] Add getting GitLab repos for user. --- R/EngineGraphQLGitLab.R | 36 +++++++++---- R/GQLQueryGitLab.R | 115 +++++++++++++++++++++++++--------------- R/GitHost.R | 21 +------- R/GitHostGitHub.R | 19 +++++++ R/GitHostGitLab.R | 24 ++++++++- 5 files changed, 138 insertions(+), 77 deletions(-) diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 7dc0b38a..e9c5d025 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -46,19 +46,22 @@ EngineGraphQLGitLab <- R6::R6Class( }, # Iterator over pulling pages of repositories. - get_repos_from_org = function(org = NULL) { + get_repos_from_org = function(org = NULL, + type = c("organization", "user")) { full_repos_list <- list() next_page <- TRUE repo_cursor <- "" while (next_page) { repos_response <- private$get_repos_page( org = org, + type = type, repo_cursor = repo_cursor ) - if (length(repos_response$data$group) == 0) { - cli::cli_abort("Empty") + core_response <- if (type == "organization") { + repos_response$data$group$projects + } else { + repos_response$data$projects } - core_response <- repos_response$data$group$projects repos_list <- core_response$edges next_page <- core_response$pageInfo$hasNextPage if (is.null(next_page)) next_page <- FALSE @@ -259,14 +262,25 @@ EngineGraphQLGitLab <- R6::R6Class( # Wrapper over building GraphQL query and response. get_repos_page = function(org = NULL, + type = "organization", repo_cursor = "") { - repos_query <- self$gql_query$repos_by_org( - repo_cursor = repo_cursor - ) - response <- self$gql_response( - gql_query = repos_query, - vars = list("org" = org) - ) + if (type == "organization") { + response <- self$gql_response( + gql_query = self$gql_query$repos_by_org(), + vars = list( + "org" = org, + "repo_cursor" = repo_cursor + ) + ) + } else { + response <- self$gql_response( + gql_query = self$gql_query$repos_by_user(), + vars = list( + "username" = org, + "repo_cursor" = repo_cursor + ) + ) + } return(response) }, diff --git a/R/GQLQueryGitLab.R b/R/GQLQueryGitLab.R index e6777dc6..6f6270bf 100644 --- a/R/GQLQueryGitLab.R +++ b/R/GQLQueryGitLab.R @@ -23,54 +23,44 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab", }' }, + #' @field user_or_org_query Query to check if a given string is user or + #' organization. + user_or_org_query = + ' + query ($username: String! $grouppath: ID!) { + user(username: $username) { + __typename + username + } + group(fullPath: $grouppath) { + __typename + fullPath + } + }' + , + #' @description Prepare query to get repositories from GitLab. - #' @param repo_cursor An end cursor for repositories page. #' @return A query. - repos_by_org = function(repo_cursor = "") { - if (nchar(repo_cursor) == 0) { - after_cursor <- repo_cursor - } else { - after_cursor <- paste0('after: "', repo_cursor, '" ') - } - paste0('query GetReposByOrg($org: ID!) { - group(fullPath: $org) { - projects(first: 100 ', after_cursor, ') { - count - pageInfo { - hasNextPage - endCursor - } - edges { - node { - repo_id: id - repo_name: name - repo_path: path - ... on Project { - repository { - rootRef - } - } - stars: starCount - forks: forksCount - created_at: createdAt - last_activity_at: lastActivityAt - languages { - name - } - issues: issueStatusCounts { - all - closed - opened - } - group { - path - } - repo_url: webUrl - } + repos_by_org = function() { + paste0(' + query GetReposByOrg($org: ID! $repo_cursor: String!) { + group(fullPath: $org) { + projects(first: 100 after: $repo_cursor) { + ', private$projects_field_content, ' } } - } - }') + }') + }, + + #' @description Prepare query to get repositories from GitLab. + #' @return A query. + repos_by_user = function() { + paste0(' + query GetUserRepos ($username: String! $repo_cursor: String!) { + projects(search: $username searchNamespaces: true after: $repo_cursor first: 100) { + ', private$projects_field_content, ' + } + }') }, #' @description Prepare query to get info on a GitLab user. @@ -216,5 +206,42 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab", } }' } + ), + private = list( + projects_field_content = + ' + count + pageInfo { + hasNextPage + endCursor + } + edges { + node { + repo_id: id + repo_name: name + repo_path: path + ... on Project { + repository { + rootRef + } + } + stars: starCount + forks: forksCount + created_at: createdAt + last_activity_at: lastActivityAt + languages { + name + } + issues: issueStatusCounts { + all + closed + opened + } + namespace { + path + } + repo_url: webUrl + } + }' ) ) diff --git a/R/GitHost.R b/R/GitHost.R index e390d3a1..c0fd164f 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -421,25 +421,6 @@ GitHost <- R6::R6Class( } }, - # Set owner type - set_owner_type = function(owners) { - graphql_engine <- private$engines$graphql - user_or_org_query <- graphql_engine$gql_query$user_or_org_query - login_types <- purrr::map(owners, function(owner) { - response <- graphql_engine$gql_response( - gql_query = user_or_org_query, - vars = list( - "login" = owner - ) - ) - type <- purrr::discard(response$data, is.null) %>% - names() - attr(owner, "type") <- type - return(owner) - }) - return(login_types) - }, - # Check if repositories exist check_repositories = function(repos, verbose) { if (verbose) { @@ -614,7 +595,7 @@ GitHost <- R6::R6Class( } graphql_engine <- private$engines$graphql repos_table <- purrr::map(private$orgs, function(org) { - type <- attr(org, "type") + type <- attr(org, "type") %||% "organization" org <- utils::URLdecode(org) if (!private$scan_all && verbose) { show_message( diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 37942bee..916892ed 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -88,6 +88,25 @@ GitHostGitHub <- R6::R6Class( private$endpoints$repositories <- glue::glue("{private$api_url}/repos") }, + # Set owner type + set_owner_type = function(owners) { + graphql_engine <- private$engines$graphql + user_or_org_query <- graphql_engine$gql_query$user_or_org_query + login_types <- purrr::map(owners, function(owner) { + response <- graphql_engine$gql_response( + gql_query = user_or_org_query, + vars = list( + "login" = owner + ) + ) + type <- purrr::discard(response$data, is.null) %>% + names() + attr(owner, "type") <- type + return(owner) + }) + return(login_types) + }, + # Setup REST and GraphQL engines setup_engines = function() { private$engines$rest <- EngineRestGitHub$new( diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index bf1ffe45..7b702034 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -147,6 +147,26 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", private$endpoints$repositories <- glue::glue("{private$api_url}/projects") }, + # Set owner type + set_owner_type = function(owners) { + graphql_engine <- private$engines$graphql + user_or_org_query <- graphql_engine$gql_query$user_or_org_query + login_types <- purrr::map(owners, function(owner) { + response <- graphql_engine$gql_response( + gql_query = user_or_org_query, + vars = list( + "username" = owner, + "grouppath" = owner + ) + ) + type <- purrr::discard(response$data, is.null) %>% + names() + attr(owner, "type") <- type + return(owner) + }) + return(login_types) + }, + # Setup REST and GraphQL engines setup_engines = function() { private$engines$rest <- EngineRestGitLab$new( @@ -220,8 +240,8 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", repo$issues_closed <- repo$issues$closed repo$issues <- NULL repo$last_activity_at <- as.POSIXct(repo$last_activity_at) - repo$organization <- repo$group$path - repo$group <- NULL + repo$organization <- repo$namespace$path + repo$namespace <- NULL repo$repo_path <- NULL # temporary to close issue 338 data.frame(repo) }) %>% From a06f3714e35ac060dcc49efc619b842636aedc68 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 10 Oct 2024 09:51:48 +0000 Subject: [PATCH 21/94] Update NEWS. --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 24fc24b9..70f0ac66 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ # GitStats (development version) +- Added possibility to get repositories for individual users with `get_repos()` ([#492](https://github.com/r-world-devs/GitStats/issues/492)). Earlier this was only possible for GitHub organizations and GitLab groups. - Fixed getting large search responses for GitHub ([#491](https://github.com/r-world-devs/GitStats/issues/491)). # GitStats 2.1.0 From 325c05efb8a48e6ac13eddd691960e3457084225 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 10 Oct 2024 11:32:29 +0000 Subject: [PATCH 22/94] Fix code and tests. --- R/EngineGraphQLGitHub.R | 5 +++-- R/EngineGraphQLGitLab.R | 3 ++- R/test_helpers.R | 12 ++++++++++-- tests/testthat/_snaps/01-get_repos-GitHub.md | 2 +- tests/testthat/_snaps/01-get_repos-GitLab.md | 2 +- tests/testthat/helper-fixtures.R | 2 +- tests/testthat/test-01-get_repos-GitHub.R | 6 ++++-- tests/testthat/test-01-get_repos-GitLab.R | 14 +++++++++----- 8 files changed, 31 insertions(+), 15 deletions(-) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 80ba97a3..d8b75bea 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -169,7 +169,7 @@ EngineGraphQLGitHub <- R6::R6Class( # Wrapper over building GraphQL query and response. get_repos_page = function(login = NULL, - type, + type = c("organization", "user"), repo_cursor = "") { repos_query <- if (type == "organization") { self$gql_query$repos_by_org() @@ -239,7 +239,8 @@ EngineGraphQLGitHub <- R6::R6Class( get_repos_data = function(org, repos = NULL) { repos_list <- self$get_repos_from_org( - org = org + org = org, + type = "organization" ) if (!is.null(repos)) { repos_list <- purrr::keep(repos_list, ~ .$repo_name %in% repos) diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index e9c5d025..eae77ad3 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -293,7 +293,8 @@ EngineGraphQLGitLab <- R6::R6Class( get_repos_data = function(org, repos = NULL) { repos_list <- self$get_repos_from_org( - org = org + org = org, + type = "organization" ) if (!is.null(repos)) { repos_list <- purrr::keep(repos_list, ~ .$node$repo_path %in% repos) diff --git a/R/test_helpers.R b/R/test_helpers.R index 75a8e38d..656c250b 100644 --- a/R/test_helpers.R +++ b/R/test_helpers.R @@ -44,7 +44,11 @@ GitHostGitHubTest <- R6::R6Class( private = list( set_orgs_and_repos_mocked = function(orgs, repos) { private$orgs <- orgs - private$repos <- repos + if (!is.null(repos)) { + private$repos <- repos + orgs_repos <- private$extract_repos_and_orgs(repos) + private$orgs <- names(orgs_repos) + } }, setup_test_engines = function() { private$engines$rest <- TestEngineRestGitHub$new( @@ -84,7 +88,11 @@ GitHostGitLabTest <- R6::R6Class( private = list( set_orgs_and_repos_mocked = function(orgs, repos) { private$orgs <- orgs - private$repos <- repos + if (!is.null(repos)) { + private$repos <- repos + orgs_repos <- private$extract_repos_and_orgs(repos) + private$orgs <- names(orgs_repos) + } }, setup_test_engines = function() { private$engines$rest <- TestEngineRestGitLab$new( diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index ed8eeb9b..0c71258f 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -3,7 +3,7 @@ Code gh_repos_by_org_query Output - [1] "\n query GetReposByOrg($org: String!) {\n repositoryOwner(login: $org) {\n ... on Organization {\n repositories(first: 100 ) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n }\n }\n }" + [1] "\n query GetReposByOrg($login: String! $repoCursor: String!) {\n repositoryOwner(login: $login) {\n ... on Organization {\n \n repositories(first: 100 after: $repoCursor) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n \n }\n }\n }" # `prepare_repos_table()` prepares repos table diff --git a/tests/testthat/_snaps/01-get_repos-GitLab.md b/tests/testthat/_snaps/01-get_repos-GitLab.md index 5335a073..c6dab9da 100644 --- a/tests/testthat/_snaps/01-get_repos-GitLab.md +++ b/tests/testthat/_snaps/01-get_repos-GitLab.md @@ -3,7 +3,7 @@ Code gl_repos_by_org_query Output - [1] "query GetReposByOrg($org: ID!) {\n group(fullPath: $org) {\n projects(first: 100 ) {\n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n group {\n path\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" + [1] "\n query GetReposByOrg($org: ID! $repo_cursor: String!) {\n group(fullPath: $org) {\n projects(first: 100 after: $repo_cursor) {\n \n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n namespace {\n path\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" # GitHost prepares table from GitLab repositories response diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 9e673f50..78019509 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -163,7 +163,7 @@ gitlab_project_node <- list( "closed" = 8, "opened" = 2 ), - "group" = list( + "namespace" = list( "path" = "test_group" ), "repo_url" = "https://test_gitlab_url.com" diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 50437b1d..047b4dcc 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -14,7 +14,8 @@ test_that("`get_repos_page()` pulls repos page from GitHub organization", { test_fixtures$github_repos_by_org_response ) gh_repos_page <- test_graphql_github_priv$get_repos_page( - org = "test_org" + login = "test_org", + type = "organization" ) expect_gh_repos_gql_response( gh_repos_page @@ -29,7 +30,8 @@ test_that("`get_repos_from_org()` prepares formatted list", { test_mocker$use("gh_repos_page") ) gh_repos_from_org <- test_graphql_github$get_repos_from_org( - org = "test_org" + org = "test_org", + type = "organization" ) expect_list_contains( gh_repos_from_org[[1]], diff --git a/tests/testthat/test-01-get_repos-GitLab.R b/tests/testthat/test-01-get_repos-GitLab.R index 67846f96..dac1588e 100644 --- a/tests/testthat/test-01-get_repos-GitLab.R +++ b/tests/testthat/test-01-get_repos-GitLab.R @@ -14,7 +14,8 @@ test_that("`get_repos_page()` pulls repos page from GitLab group", { test_fixtures$gitlab_repos_by_org_response ) gl_repos_page <- test_graphql_gitlab_priv$get_repos_page( - org = "test_org" + org = "test_org", + type = "organization" ) expect_gl_repos_gql_response( gl_repos_page @@ -29,14 +30,15 @@ test_that("`get_repos_from_org()` prepares formatted list", { test_mocker$use("gl_repos_page") ) gl_repos_from_org <- test_graphql_gitlab$get_repos_from_org( - org = "test_org" + org = "test_org", + type = "organization" ) expect_equal( names(gl_repos_from_org[[1]]$node), c( "repo_id", "repo_name", "repo_path", "repository", "stars", "forks", "created_at", "last_activity_at", - "languages", "issues", "group", "repo_url" + "languages", "issues", "namespace", "repo_url" ) ) test_mocker$cache(gl_repos_from_org) @@ -49,7 +51,8 @@ test_that("`get_repos_from_org()` does not fail when GraphQL response is not com test_fixtures$empty_gql_response ) gl_repos_from_org <- test_graphql_gitlab$get_repos_from_org( - org = "test_org" + org = "test_org", + type = "organization" ) expect_type( gl_repos_from_org, @@ -65,7 +68,8 @@ test_that("`get_repos_from_org()` does not fail when GraphQL response is not com test_fixtures$half_empty_gql_response ) gl_repos_from_org <- test_graphql_gitlab$get_repos_from_org( - org = "test_org" + org = "test_org", + type = "organization" ) expect_type( gl_repos_from_org, From 6186fe54af0b928d5aacdc2051f6000c31882c57 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 10 Oct 2024 11:36:19 +0000 Subject: [PATCH 23/94] Update docs. --- R/gitstats_functions.R | 6 +++--- man/get_repos.Rd | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index d98afac4..a4cbd1c2 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -89,9 +89,9 @@ set_gitlab_host <- function(gitstats_object, #' @title Get data on repositories #' @name get_repos -#' @description Pulls data on all repositories for an organization or those -#' with a given text in code blobs (`with_code` parameter) or a file -#' (`with_files` parameter) and parse it into table format. +#' @description Pulls data on all repositories for an organization, individual +#' user or those with a given text in code blobs (`with_code` parameter) or a +#' file (`with_files` parameter) and parse it into table format. #' @param gitstats_object A GitStats object. #' @param add_contributors A logical parameter to decide whether to add #' information about repositories' contributors to the repositories output diff --git a/man/get_repos.Rd b/man/get_repos.Rd index ac33c371..6bf5406b 100644 --- a/man/get_repos.Rd +++ b/man/get_repos.Rd @@ -52,9 +52,9 @@ output is switched off.} A data.frame. } \description{ -Pulls data on all repositories for an organization or those -with a given text in code blobs (\code{with_code} parameter) or a file -(\code{with_files} parameter) and parse it into table format. +Pulls data on all repositories for an organization, individual +user or those with a given text in code blobs (\code{with_code} parameter) or a +file (\code{with_files} parameter) and parse it into table format. } \examples{ \dontrun{ From 4fd12cc6d55182db93927f1cd819f8f6a8cfdb62 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 10 Oct 2024 12:06:05 +0000 Subject: [PATCH 24/94] Add tests to cover new feature. --- tests/testthat/helper-expect-responses.R | 18 +++++++--- tests/testthat/helper-fixtures.R | 40 +++++++++++++++++++++++ tests/testthat/test-01-get_repos-GitHub.R | 38 +++++++++++++++++++++ tests/testthat/test-01-get_repos-GitLab.R | 38 +++++++++++++++++++++ 4 files changed, 130 insertions(+), 4 deletions(-) diff --git a/tests/testthat/helper-expect-responses.R b/tests/testthat/helper-expect-responses.R index 798275e3..a0590472 100644 --- a/tests/testthat/helper-expect-responses.R +++ b/tests/testthat/helper-expect-responses.R @@ -48,20 +48,25 @@ expect_gh_repos_rest_response <- function(object) { }) } -expect_gl_repos_gql_response <- function(object) { +expect_gl_repos_gql_response <- function(object, type = "organization") { expect_type( object, "list" ) + repo_node <- if (type == "organization") { + object$data$group$projects$edges[[1]]$node + } else { + object$data$projects$edges[[1]]$node + } expect_list_contains( - object$data$group$projects$edges[[1]]$node, + repo_node, c( "id", "name", "repository", "stars", "forks", "created_at", "last_activity_at" ) ) } -expect_gh_repos_gql_response <- function(object) { +expect_gh_repos_gql_response <- function(object, type = "organization") { expect_type( object, "list" @@ -70,8 +75,13 @@ expect_gh_repos_gql_response <- function(object) { object, "data" ) + repo_node <- if (type == "organization") { + object$data$repositoryOwner$repositories$nodes[[1]] + } else { + object$data$user$repositories$nodes[[1]] + } expect_list_contains( - object$data$repositoryOwner$repositories$nodes[[1]], + repo_node, c( "id", "name", "stars", "forks", "created_at", "last_activity_at", "languages", "issues_open", "issues_closed", diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 78019509..6d574213 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -138,6 +138,27 @@ test_fixtures$github_repos_by_org_response <- list( ) ) +test_fixtures$github_repos_by_user_response <- list( + "data" = list( + "user" = list( + "repositories" = list( + "totalCount" = 5, + "pageInfo" = list( + "endCursor" = "xyx", + "hasNextPage" = FALSE + ), + "nodes" = list( + github_repository_node, + github_repository_node, + github_repository_node, + github_repository_node, + github_repository_node + ) + ) + ) + ) +) + gitlab_project_node <- list( "node" = list( "repo_id" = "gid://gitlab/Project/61399846", @@ -191,6 +212,25 @@ test_fixtures$gitlab_repos_by_org_response <- list( ) ) +test_fixtures$gitlab_repos_by_user_response <- list( + "data" = list( + "projects" = list( + "count" = 5, + "pageInfo" = list( + "hasNextPage" = FALSE, + "endCursor" = "xyz" + ), + "edges" = list( + gitlab_project_node, + gitlab_project_node, + gitlab_project_node, + gitlab_project_node, + gitlab_project_node + ) + ) + ) +) + github_commit_edge <- list( "node" = list( "id" = "xxx", diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 047b4dcc..3c09ab50 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -44,6 +44,44 @@ test_that("`get_repos_from_org()` prepares formatted list", { test_mocker$cache(gh_repos_from_org) }) +test_that("`get_repos_page()` pulls repos page from GitHub user", { + mockery::stub( + test_graphql_github_priv$get_repos_page, + "self$gql_response", + test_fixtures$github_repos_by_user_response + ) + gh_repos_user_page <- test_graphql_github_priv$get_repos_page( + login = "test_user", + type = "user" + ) + expect_gh_repos_gql_response( + gh_repos_user_page, + type = "user" + ) + test_mocker$cache(gh_repos_user_page) +}) + +test_that("`get_repos_from_org()` prepares formatted list", { + mockery::stub( + test_graphql_github$get_repos_from_org, + "private$get_repos_page", + test_mocker$use("gh_repos_user_page") + ) + gh_repos_from_user <- test_graphql_github$get_repos_from_org( + org = "test_user", + type = "user" + ) + expect_list_contains( + gh_repos_from_user[[1]], + c( + "id", "name", "stars", "forks", "created_at", + "last_activity_at", "languages", "issues_open", "issues_closed", + "contributors", "repo_url" + ) + ) + test_mocker$cache(gh_repos_from_user) +}) + # REST Engine search repos by code test_that("`search_response()` performs search with limit under 1000", { diff --git a/tests/testthat/test-01-get_repos-GitLab.R b/tests/testthat/test-01-get_repos-GitLab.R index dac1588e..0fbe209f 100644 --- a/tests/testthat/test-01-get_repos-GitLab.R +++ b/tests/testthat/test-01-get_repos-GitLab.R @@ -44,6 +44,44 @@ test_that("`get_repos_from_org()` prepares formatted list", { test_mocker$cache(gl_repos_from_org) }) +test_that("`get_repos_page()` pulls repos page from GitLab user", { + mockery::stub( + test_graphql_gitlab_priv$get_repos_page, + "self$gql_response", + test_fixtures$gitlab_repos_by_user_response + ) + gl_repos_user_page <- test_graphql_gitlab_priv$get_repos_page( + org = "test_user", + type = "user" + ) + expect_gl_repos_gql_response( + gl_repos_user_page, + type = "user" + ) + test_mocker$cache(gl_repos_user_page) +}) + +test_that("`get_repos_from_org()` prepares formatted list", { + mockery::stub( + test_graphql_gitlab$get_repos_from_org, + "private$get_repos_page", + test_mocker$use("gl_repos_user_page") + ) + gl_repos_from_user <- test_graphql_gitlab$get_repos_from_org( + org = "test_user", + type = "user" + ) + expect_equal( + names(gl_repos_from_user[[1]]$node), + c( + "repo_id", "repo_name", "repo_path", "repository", + "stars", "forks", "created_at", "last_activity_at", + "languages", "issues", "namespace", "repo_url" + ) + ) + test_mocker$cache(gl_repos_from_user) +}) + test_that("`get_repos_from_org()` does not fail when GraphQL response is not complete", { mockery::stub( test_graphql_gitlab$get_repos_from_org, From e1cbbcc8da418f35107cefbe65c2c959f2ee5e44 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 10 Oct 2024 12:50:17 +0000 Subject: [PATCH 25/94] Adjust getting files to new feature. --- R/EngineGraphQLGitHub.R | 8 +- R/EngineGraphQLGitLab.R | 143 ++++++++++-------- R/GitHost.R | 4 + .../_snaps/get_files_content-GitLab.md | 6 +- .../testthat/test-get_files_content-GitLab.R | 2 + 5 files changed, 96 insertions(+), 67 deletions(-) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index d8b75bea..ed03264f 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -93,6 +93,7 @@ EngineGraphQLGitHub <- R6::R6Class( # Pull all given files from all repositories of an organization. get_files_from_org = function(org, + type, repos, file_paths, host_files_structure, @@ -101,6 +102,7 @@ EngineGraphQLGitHub <- R6::R6Class( progress = TRUE) { repo_data <- private$get_repos_data( org = org, + type = type, repos = repos ) repositories <- repo_data[["repositories"]] @@ -123,6 +125,7 @@ EngineGraphQLGitHub <- R6::R6Class( # Pull all files from all repositories of an organization. get_files_structure_from_org = function(org, + type, repos, pattern = NULL, depth = Inf, @@ -130,6 +133,7 @@ EngineGraphQLGitHub <- R6::R6Class( progress = TRUE) { repo_data <- private$get_repos_data( org = org, + type = type, repos = repos ) repositories <- repo_data[["repositories"]] @@ -237,10 +241,10 @@ EngineGraphQLGitHub <- R6::R6Class( return(response) }, - get_repos_data = function(org, repos = NULL) { + get_repos_data = function(org, type, repos = NULL) { repos_list <- self$get_repos_from_org( org = org, - type = "organization" + type = type ) if (!is.null(repos)) { repos_list <- purrr::keep(repos_list, ~ .$repo_name %in% repos) diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index eae77ad3..ccf47280 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -84,6 +84,7 @@ EngineGraphQLGitLab <- R6::R6Class( # pulled files_structure. In such a case GitStats will switch from this function # to iterator over repositories (multiple queries), as it is done for GitHub. get_files_from_org = function(org, + type, repos, file_paths, host_files_structure, @@ -103,73 +104,87 @@ EngineGraphQLGitLab <- R6::R6Class( } else if (is.null(host_files_structure) && only_text_files) { file_paths <- file_paths[!grepl(non_text_files_pattern, file_paths)] } - while (next_page) { - files_query <- self$gql_query$files_by_org( - end_cursor = end_cursor - ) - files_response <- tryCatch( - { - self$gql_response( - gql_query = files_query, - vars = list( - "org" = org, - "file_paths" = file_paths + if (type == "organization") { + while (next_page) { + files_query <- self$gql_query$files_by_org( + end_cursor = end_cursor + ) + files_response <- tryCatch( + { + self$gql_response( + gql_query = files_query, + vars = list( + "org" = org, + "file_paths" = file_paths + ) ) - ) - }, - error = function(e) { - list() - } - ) - if (private$is_query_error(files_response)) { - if (verbose) { - purrr::walk(files_response$errors, ~ cli::cli_alert_warning(.)) - } - if (private$is_complexity_error(files_response)) { + }, + error = function(e) { + list() + } + ) + if (private$is_query_error(files_response)) { if (verbose) { - cli::cli_alert_info( - cli::col_br_cyan("I will switch to pulling files per repository.") + purrr::walk(files_response$errors, ~ cli::cli_alert_warning(.)) + } + if (private$is_complexity_error(files_response)) { + if (verbose) { + cli::cli_alert_info( + cli::col_br_cyan("I will switch to pulling files per repository.") + ) + } + full_files_list <- self$get_files_from_org_per_repo( + org = org, + type = type, + repos = repos, + file_paths = file_paths, + host_files_structure = host_files_structure, + only_text_files = only_text_files, + verbose = verbose, + progress = progress ) + return(full_files_list) } - full_files_list <- self$get_files_from_org_per_repo( - org = org, - repos = repos, - file_paths = file_paths, - host_files_structure = host_files_structure, - only_text_files = only_text_files, - verbose = verbose, - progress = progress - ) - return(full_files_list) } + if (length(files_response$data$group) == 0 && verbose) { + cli::cli_alert_danger("Empty response.") + } + projects <- files_response$data$group$projects + files_list <- purrr::map(projects$edges, function(edge) { + edge$node + }) %>% + purrr::discard(~ length(.$repository$blobs$nodes) == 0) + if (is.null(files_list)) files_list <- list() + if (length(files_list) > 0) { + next_page <- files_response$pageInfo$hasNextPage + } else { + next_page <- FALSE + } + if (is.null(next_page)) next_page <- FALSE + if (next_page) { + end_cursor <- files_response$pageInfo$endCursor + } else { + end_cursor <- "" + } + full_files_list <- append(full_files_list, files_list) } - if (length(files_response$data$group) == 0 && verbose) { - cli::cli_alert_danger("Empty response.") - } - projects <- files_response$data$group$projects - files_list <- purrr::map(projects$edges, function(edge) { - edge$node - }) %>% - purrr::discard(~ length(.$repository$blobs$nodes) == 0) - if (is.null(files_list)) files_list <- list() - if (length(files_list) > 0) { - next_page <- files_response$pageInfo$hasNextPage - } else { - next_page <- FALSE + if (!is.null(repos)) { + full_files_list <- purrr::keep(full_files_list, function(project) { + repo_name <- private$get_repo_name_from_url(project$webUrl) + repo_name %in% repos + }) } - if (is.null(next_page)) next_page <- FALSE - if (next_page) { - end_cursor <- files_response$pageInfo$endCursor - } else { - end_cursor <- "" - } - full_files_list <- append(full_files_list, files_list) - } - if (!is.null(repos)) { - full_files_list <- purrr::keep(full_files_list, function(project) { - repo_name <- private$get_repo_name_from_url(project$webUrl) - repo_name %in% repos - }) + } else { + full_files_list <- self$get_files_from_org_per_repo( + org = org, + type = type, + repos = repos, + file_paths = file_paths, + host_files_structure = host_files_structure, + only_text_files = only_text_files, + verbose = verbose, + progress = progress + ) } return(full_files_list) }, @@ -178,6 +193,7 @@ EngineGraphQLGitLab <- R6::R6Class( # one query way applied with get_files_from_org() fails due to its complexity. # For more info see docs above. get_files_from_org_per_repo = function(org, + type, repos, file_paths = NULL, host_files_structure = NULL, @@ -187,6 +203,7 @@ EngineGraphQLGitLab <- R6::R6Class( if (is.null(repos)) { repo_data <- private$get_repos_data( org = org, + type = type, repos = repos ) repos <- repo_data[["repositories"]] @@ -216,6 +233,7 @@ EngineGraphQLGitLab <- R6::R6Class( return(org_files_list) }, get_files_structure_from_org = function(org, + type, repos, pattern = NULL, depth = Inf, @@ -223,6 +241,7 @@ EngineGraphQLGitLab <- R6::R6Class( progress = TRUE) { repo_data <- private$get_repos_data( org = org, + type = type, repos = repos ) repositories <- repo_data[["repositories"]] @@ -291,10 +310,10 @@ EngineGraphQLGitLab <- R6::R6Class( return(repo_name) }, - get_repos_data = function(org, repos = NULL) { + get_repos_data = function(org, type, repos = NULL) { repos_list <- self$get_repos_from_org( org = org, - type = "organization" + type = type ) if (!is.null(repos)) { repos_list <- purrr::keep(repos_list, ~ .$node$repo_path %in% repos) diff --git a/R/GitHost.R b/R/GitHost.R index c0fd164f..9b552fd8 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -886,8 +886,10 @@ GitHost <- R6::R6Class( information = user_msg ) } + type <- attr(org, "type") %||% "organization" graphql_engine$get_files_from_org( org = org, + type = type, repos = repos, file_paths = file_path, host_files_structure = host_files_structure, @@ -932,8 +934,10 @@ GitHost <- R6::R6Class( information = user_info ) } + type <- attr(org, "type") %||% "organization" graphql_engine$get_files_structure_from_org( org = org, + type = type, repos = private$repos, pattern = pattern, depth = depth, diff --git a/tests/testthat/_snaps/get_files_content-GitLab.md b/tests/testthat/_snaps/get_files_content-GitLab.md index 3dcd9476..076446d5 100644 --- a/tests/testthat/_snaps/get_files_content-GitLab.md +++ b/tests/testthat/_snaps/get_files_content-GitLab.md @@ -16,9 +16,9 @@ Code gitlab_files_response_by_repos <- test_graphql_gitlab$get_files_from_org(org = "mbtests", - repos = NULL, file_paths = c("DESCRIPTION", "project_metadata.yaml", - "README.md"), host_files_structure = NULL, only_text_files = TRUE, verbose = TRUE, - progress = FALSE) + type = "organization", repos = NULL, file_paths = c("DESCRIPTION", + "project_metadata.yaml", "README.md"), host_files_structure = NULL, + only_text_files = TRUE, verbose = TRUE, progress = FALSE) Message i I will switch to pulling files per repository. diff --git a/tests/testthat/test-get_files_content-GitLab.R b/tests/testthat/test-get_files_content-GitLab.R index 9ea69b2e..c034666b 100644 --- a/tests/testthat/test-get_files_content-GitLab.R +++ b/tests/testthat/test-get_files_content-GitLab.R @@ -57,6 +57,7 @@ test_that("GitLab GraphQL Engine pulls files from a group", { ) gitlab_files_response <- test_graphql_gitlab$get_files_from_org( org = "mbtests", + type = "organization", repos = NULL, file_paths = "meta_data.yaml", only_text_files = TRUE, @@ -113,6 +114,7 @@ test_that("Gitlab GraphQL switches to pulling files per repositories when query expect_snapshot( gitlab_files_response_by_repos <- test_graphql_gitlab$get_files_from_org( org = "mbtests", + type = "organization", repos = NULL, file_paths = c("DESCRIPTION", "project_metadata.yaml", "README.md"), host_files_structure = NULL, From b6b214e0dc7a4626c7f0721099c9e0b1f920e003 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 10 Oct 2024 16:02:54 +0000 Subject: [PATCH 26/94] Add publishing of code coverage. --- .github/workflows/test-coverage.yaml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 0edc999d..49d370bc 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -1,5 +1,3 @@ -# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples -# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help on: push: branches: [master] @@ -14,6 +12,7 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT}} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} USE_RENV: "FALSE" steps: @@ -28,7 +27,7 @@ jobs: extra-packages: any::covr needs: coverage - - name: Test coverage + - name: Code coverage run: | covr::codecov( quiet = FALSE, @@ -37,6 +36,15 @@ jobs: ) shell: Rscript {0} + - name: Upload coverage report to Codecov + uses: codecov/codecov-action@v2 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: true + - name: Show testthat output if: always() run: | From 7059be5f16c1154410b4f2fd794411a0517f627e Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 11 Oct 2024 08:26:22 +0000 Subject: [PATCH 27/94] Try fix workflow. --- .github/workflows/test-coverage.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 49d370bc..2f02eda1 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -40,7 +40,6 @@ jobs: uses: codecov/codecov-action@v2 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml flags: unittests name: codecov-umbrella fail_ci_if_error: true From 30fa9078492396e436852254e10036f7c0815fbe Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 11 Oct 2024 08:45:15 +0000 Subject: [PATCH 28/94] Add example workflow. --- inst/get_repos_for_individual_users.R | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 inst/get_repos_for_individual_users.R diff --git a/inst/get_repos_for_individual_users.R b/inst/get_repos_for_individual_users.R new file mode 100644 index 00000000..03a60a7f --- /dev/null +++ b/inst/get_repos_for_individual_users.R @@ -0,0 +1,11 @@ +create_gitstats() %>% + set_github_host( + repos = "ddsjoberg/gtsummary" + ) %>% + get_repos() + +create_gitstats() %>% + set_github_host( + repos = "hadley/elmer" + ) %>% + get_repos() From d34a6212f1dc333db90543ba972b0ed5f85b93e9 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 11 Oct 2024 08:53:05 +0000 Subject: [PATCH 29/94] Try generate codecov report. --- .github/workflows/test-coverage.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 2f02eda1..93349055 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -36,10 +36,15 @@ jobs: ) shell: Rscript {0} + - name: Generate code coverage report + run: | + Rscript -e "covr::report(covr::package_coverage(), file='coverage.xml')" + - name: Upload coverage report to Codecov uses: codecov/codecov-action@v2 with: token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml flags: unittests name: codecov-umbrella fail_ci_if_error: true From 674be9b817b66d34b4ef5c1bdc77a1f32b4d204e Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 11 Oct 2024 09:13:42 +0000 Subject: [PATCH 30/94] Update renv. --- renv.lock | 467 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 450 insertions(+), 17 deletions(-) diff --git a/renv.lock b/renv.lock index 26ad6a5d..dabdc725 100644 --- a/renv.lock +++ b/renv.lock @@ -50,6 +50,13 @@ ], "Hash": "543776ae6848fde2f48ff3816d0628bc" }, + "brew": { + "Package": "brew", + "Version": "1.0-10", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "8f4a384e19dccd8c65356dc096847b76" + }, "brio": { "Package": "brio", "Version": "1.1.4", @@ -107,14 +114,24 @@ }, "cli": { "Package": "cli", - "Version": "3.6.2", + "Version": "3.6.3", "Source": "Repository", "Repository": "CRAN", "Requirements": [ "R", "utils" ], - "Hash": "1216ac65ac55ec0058a6f75d7ca0fd52" + "Hash": "b21916dd77a27642b447374a5d30ecf3" + }, + "clipr": { + "Package": "clipr", + "Version": "0.8.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "utils" + ], + "Hash": "3f038e5ac7f41d4ac41ce658c85e3042" }, "commonmark": { "Package": "commonmark", @@ -145,6 +162,20 @@ ], "Hash": "e8a1e41acf02548751f45c718d55aa6a" }, + "credentials": { + "Package": "credentials", + "Version": "2.0.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "askpass", + "curl", + "jsonlite", + "openssl", + "sys" + ], + "Hash": "c7844b32098dcbd1c59cbd8dddb4ecc6" + }, "curl": { "Package": "curl", "Version": "5.2.1", @@ -168,6 +199,40 @@ ], "Hash": "99b79fcbd6c4d1ce087f5c5c758b384f" }, + "devtools": { + "Package": "devtools", + "Version": "2.4.5", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "desc", + "ellipsis", + "fs", + "lifecycle", + "memoise", + "miniUI", + "pkgbuild", + "pkgdown", + "pkgload", + "profvis", + "rcmdcheck", + "remotes", + "rlang", + "roxygen2", + "rversions", + "sessioninfo", + "stats", + "testthat", + "tools", + "urlchecker", + "usethis", + "utils", + "withr" + ], + "Hash": "ea5bc8b4a6a01e4f12d98b58329930bb" + }, "diffobj": { "Package": "diffobj", "Version": "0.3.5", @@ -237,6 +302,17 @@ ], "Hash": "fedd9d00c2944ff00a0e2696ccf048ec" }, + "ellipsis": { + "Package": "ellipsis", + "Version": "0.3.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "rlang" + ], + "Hash": "bb0eec2fe32e88d9e2836c2f73ea2077" + }, "evaluate": { "Package": "evaluate", "Version": "0.23", @@ -301,33 +377,76 @@ ], "Hash": "15e9634c0fcd294799e9b2e929ed1b86" }, + "gert": { + "Package": "gert", + "Version": "2.0.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "askpass", + "credentials", + "openssl", + "rstudioapi", + "sys", + "zip" + ], + "Hash": "f70d3fe2d9e7654213a946963d1591eb" + }, + "gh": { + "Package": "gh", + "Version": "1.4.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "gitcreds", + "glue", + "httr2", + "ini", + "jsonlite", + "lifecycle", + "rlang" + ], + "Hash": "fbbbc48eba7a6626a08bb365e44b563b" + }, + "gitcreds": { + "Package": "gitcreds", + "Version": "0.1.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "ab08ac61f3e1be454ae21911eb8bc2fe" + }, "glue": { "Package": "glue", - "Version": "1.7.0", + "Version": "1.8.0", "Source": "Repository", "Repository": "CRAN", "Requirements": [ "R", "methods" ], - "Hash": "e0b3a53876554bd45879e596cdb10a52" + "Hash": "5899f1eaa825580172bb56c08266f37c" }, "highr": { "Package": "highr", - "Version": "0.10", + "Version": "0.11", "Source": "Repository", "Repository": "CRAN", "Requirements": [ "R", "xfun" ], - "Hash": "06230136b2d2b9ba5805e1963fa6e890" + "Hash": "d65ba49117ca223614f71b60d85b8ab7" }, "htmltools": { "Package": "htmltools", - "Version": "0.5.8", + "Version": "0.5.8.1", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "base64enc", @@ -337,7 +456,37 @@ "rlang", "utils" ], - "Hash": "149431ee39aba5bdc264112c8ff94444" + "Hash": "81d371a9cc60640e74e4ab6ac46dcedc" + }, + "htmlwidgets": { + "Package": "htmlwidgets", + "Version": "1.6.4", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "grDevices", + "htmltools", + "jsonlite", + "knitr", + "rmarkdown", + "yaml" + ], + "Hash": "04291cc45198225444a397606810ac37" + }, + "httpuv": { + "Package": "httpuv", + "Version": "1.6.15", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "R6", + "Rcpp", + "later", + "promises", + "utils" + ], + "Hash": "d55aa087c47a63ead0f6fc10f8fa1ee0" }, "httr": { "Package": "httr", @@ -387,6 +536,13 @@ ], "Hash": "e957e989ea17f937964f0d46b0f0bca0" }, + "ini": { + "Package": "ini", + "Version": "0.3.1", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "6154ec2223172bce8162d4153cda21f7" + }, "jquerylib": { "Package": "jquerylib", "Version": "0.1.4", @@ -409,7 +565,7 @@ }, "knitr": { "Package": "knitr", - "Version": "1.45", + "Version": "1.48", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -421,7 +577,18 @@ "xfun", "yaml" ], - "Hash": "1ec462871063897135c1bcbe0fc8f07d" + "Hash": "acf380f300c721da9fde7df115a5f86f" + }, + "later": { + "Package": "later", + "Version": "1.3.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "Rcpp", + "rlang" + ], + "Hash": "a3e051d405326b8b0012377434c62b37" }, "lifecycle": { "Package": "lifecycle", @@ -480,6 +647,18 @@ ], "Hash": "18e9c28c1d3ca1560ce30658b22ce104" }, + "miniUI": { + "Package": "miniUI", + "Version": "0.1.1.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "htmltools", + "shiny", + "utils" + ], + "Hash": "fec5f52652d60615fdb3957b3d74324a" + }, "mockery": { "Package": "mockery", "Version": "0.4.4", @@ -600,6 +779,16 @@ "Repository": "RSPM", "Hash": "a555924add98c99d2f411e37e7d25e9f" }, + "prettyunits": { + "Package": "prettyunits", + "Version": "1.2.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "6b01fc98b1e86c4f705ce9dcfd2f57c7" + }, "processx": { "Package": "processx", "Version": "3.8.4", @@ -613,6 +802,37 @@ ], "Hash": "0c90a7d71988856bad2a2a45dd871bb9" }, + "profvis": { + "Package": "profvis", + "Version": "0.3.8", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "htmlwidgets", + "purrr", + "rlang", + "stringr", + "vctrs" + ], + "Hash": "aa5a3864397ce6ae03458f98618395a1" + }, + "promises": { + "Package": "promises", + "Version": "1.3.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R6", + "Rcpp", + "fastmap", + "later", + "magrittr", + "rlang", + "stats" + ], + "Hash": "434cd5388a3979e74be5c219bcd6e77d" + }, "ps": { "Package": "ps", "Version": "1.7.6", @@ -660,6 +880,28 @@ ], "Hash": "5e3c5dc0b071b21fa128676560dbe94d" }, + "rcmdcheck": { + "Package": "rcmdcheck", + "Version": "1.4.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R6", + "callr", + "cli", + "curl", + "desc", + "digest", + "pkgbuild", + "prettyunits", + "rprojroot", + "sessioninfo", + "utils", + "withr", + "xopen" + ], + "Hash": "8f25ebe2ec38b1f2aef3b0d2ef76f6c4" + }, "rematch2": { "Package": "rematch2", "Version": "2.1.2", @@ -670,6 +912,20 @@ ], "Hash": "76c9e04c712a05848ae7a23d2f170a40" }, + "remotes": { + "Package": "remotes", + "Version": "2.5.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "methods", + "stats", + "tools", + "utils" + ], + "Hash": "3ee025083e66f18db6cf27b56e23e141" + }, "renv": { "Package": "renv", "Version": "1.0.5", @@ -682,18 +938,18 @@ }, "rlang": { "Package": "rlang", - "Version": "1.1.3", + "Version": "1.1.4", "Source": "Repository", "Repository": "CRAN", "Requirements": [ "R", "utils" ], - "Hash": "42548638fae05fd9a9b5f3f437fbbbe2" + "Hash": "3eec01f8b1dee337674b2e34ab1f9bc1" }, "rmarkdown": { "Package": "rmarkdown", - "Version": "2.26", + "Version": "2.28", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -712,7 +968,33 @@ "xfun", "yaml" ], - "Hash": "9b148e7f95d33aac01f31282d49e4f44" + "Hash": "062470668513dcda416927085ee9bdc7" + }, + "roxygen2": { + "Package": "roxygen2", + "Version": "7.3.2", + "Source": "Repository", + "Repository": "CRAN", + "Requirements": [ + "R", + "R6", + "brew", + "cli", + "commonmark", + "cpp11", + "desc", + "knitr", + "methods", + "pkgload", + "purrr", + "rlang", + "stringi", + "stringr", + "utils", + "withr", + "xml2" + ], + "Hash": "6ee25f9054a70f44d615300ed531ba8d" }, "rprojroot": { "Package": "rprojroot", @@ -724,6 +1006,25 @@ ], "Hash": "4c8415e0ec1e29f3f4f6fc108bef0144" }, + "rstudioapi": { + "Package": "rstudioapi", + "Version": "0.16.0", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "96710351d642b70e8f02ddeb237c46a7" + }, + "rversions": { + "Package": "rversions", + "Version": "2.1.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "curl", + "utils", + "xml2" + ], + "Hash": "a9881dfed103e83f9de151dc17002cd1" + }, "sass": { "Package": "sass", "Version": "0.4.9", @@ -738,6 +1039,62 @@ ], "Hash": "d53dbfddf695303ea4ad66f86e99b95d" }, + "sessioninfo": { + "Package": "sessioninfo", + "Version": "1.2.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "tools", + "utils" + ], + "Hash": "3f9796a8d0a0e8c6eb49a4b029359d1f" + }, + "shiny": { + "Package": "shiny", + "Version": "1.8.1.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "R6", + "bslib", + "cachem", + "commonmark", + "crayon", + "fastmap", + "fontawesome", + "glue", + "grDevices", + "htmltools", + "httpuv", + "jsonlite", + "later", + "lifecycle", + "methods", + "mime", + "promises", + "rlang", + "sourcetools", + "tools", + "utils", + "withr", + "xtable" + ], + "Hash": "54b26646816af9960a4c64d8ceec75d6" + }, + "sourcetools": { + "Package": "sourcetools", + "Version": "0.1.7-1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "5f5a7629f956619d519205ec475fe647" + }, "spelling": { "Package": "spelling", "Version": "2.3.0", @@ -896,6 +1253,51 @@ ], "Hash": "be7a76845222ad20adb761f462eed3ea" }, + "urlchecker": { + "Package": "urlchecker", + "Version": "1.0.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "curl", + "tools", + "xml2" + ], + "Hash": "409328b8e1253c8d729a7836fe7f7a16" + }, + "usethis": { + "Package": "usethis", + "Version": "2.2.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "clipr", + "crayon", + "curl", + "desc", + "fs", + "gert", + "gh", + "glue", + "jsonlite", + "lifecycle", + "purrr", + "rappdirs", + "rlang", + "rprojroot", + "rstudioapi", + "stats", + "utils", + "whisker", + "withr", + "yaml" + ], + "Hash": "d524fd42c517035027f866064417d7e6" + }, "utf8": { "Package": "utf8", "Version": "1.2.4", @@ -959,15 +1361,16 @@ }, "xfun": { "Package": "xfun", - "Version": "0.43", + "Version": "0.48", "Source": "Repository", "Repository": "CRAN", "Requirements": [ + "R", "grDevices", "stats", "tools" ], - "Hash": "ab6371d8653ce5f2f9290f4ec7b42a8e" + "Hash": "89e455b87c84e227eb7f60a1b4e5fe1f" }, "xml2": { "Package": "xml2", @@ -982,12 +1385,42 @@ ], "Hash": "1d0336142f4cd25d8d23cd3ba7a8fb61" }, + "xopen": { + "Package": "xopen", + "Version": "1.0.1", + "Source": "Repository", + "Repository": "CRAN", + "Requirements": [ + "R", + "processx" + ], + "Hash": "423df1e86d5533fcb73c6b02b4923b49" + }, + "xtable": { + "Package": "xtable", + "Version": "1.8-4", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "stats", + "utils" + ], + "Hash": "b8acdf8af494d9ec19ccb2481a9b11c2" + }, "yaml": { "Package": "yaml", "Version": "2.3.8", "Source": "Repository", "Repository": "CRAN", "Hash": "29240487a071f535f5e5d5a323b7afbd" + }, + "zip": { + "Package": "zip", + "Version": "2.3.1", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "fcc4bd8e6da2d2011eb64a5e5cc685ab" } } } From 3f6e9efff1e344e728d6889eece224b7095b292f Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 11 Oct 2024 09:22:07 +0000 Subject: [PATCH 31/94] Remove code coverage. --- .github/workflows/test-coverage.yaml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 93349055..648d9a22 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -36,19 +36,6 @@ jobs: ) shell: Rscript {0} - - name: Generate code coverage report - run: | - Rscript -e "covr::report(covr::package_coverage(), file='coverage.xml')" - - - name: Upload coverage report to Codecov - uses: codecov/codecov-action@v2 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.xml - flags: unittests - name: codecov-umbrella - fail_ci_if_error: true - - name: Show testthat output if: always() run: | From e2767b619c36218b872076f94e993ae9abefb343 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 14 Oct 2024 11:15:14 +0000 Subject: [PATCH 32/94] Fix check of token scopes and give more informative feedback when they are not sufficient. --- DESCRIPTION | 2 +- NEWS.md | 1 + R/GitHost.R | 23 ++++++++++++++++++----- R/GitHostGitHub.R | 10 +++++++--- R/GitHostGitLab.R | 36 +++++++++++++++++++++--------------- inst/set_tokens.R | 26 ++++++++++++++++++++++++++ 6 files changed, 74 insertions(+), 24 deletions(-) create mode 100644 inst/set_tokens.R diff --git a/DESCRIPTION b/DESCRIPTION index 27e860a9..971b2a0f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.0.9001 +Version: 2.1.0.9002 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), diff --git a/NEWS.md b/NEWS.md index 70f0ac66..9c432831 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ - Added possibility to get repositories for individual users with `get_repos()` ([#492](https://github.com/r-world-devs/GitStats/issues/492)). Earlier this was only possible for GitHub organizations and GitLab groups. - Fixed getting large search responses for GitHub ([#491](https://github.com/r-world-devs/GitStats/issues/491)). +- Fixed checking token scopes ([#501](https://github.com/r-world-devs/GitStats/issues/501)). If token scopes are insufficient error is returned and `GitHost` is not passed to `GitStats`. This also applies to situation when `GitStats` looks for default tokens (not defined by user). Earlier, if tests for token failed, an empty token was passed and `GitStats` was created, which was misleading for the user. # GitStats 2.1.0 diff --git a/R/GitHost.R b/R/GitHost.R index 9b552fd8..04b7e0c5 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -265,6 +265,9 @@ GitHost <- R6::R6Class( # A token. token = NULL, + # Actual token scopes + token_scopes = NULL, + # public A boolean. is_public = NULL, @@ -343,8 +346,10 @@ GitHost <- R6::R6Class( if (!private$test_token(token)) { cli::cli_abort(c( "x" = "Token exists but does not grant access.", - "i" = "Check if you use correct token. Check scopes your token is using." - )) + "i" = "Check if you use correct token.", + "!" = "Scope that is needed: [{paste0(private$min_access_scopes, collapse = ', ')}]." + ), + call = NULL) } else { return(token) } @@ -526,6 +531,14 @@ GitHost <- R6::R6Class( } } } + if (token == "") { + cli::cli_abort(c( + "x" = "No sufficient token found among: [{paste0(pat_names, collapse = ', ')}].", + "i" = "Check if you have correct token.", + "!" = "Scope that is needed: [{paste0(private$min_access_scopes, collapse = ', ')}]." + ), + call = NULL) + } return(token) }, @@ -537,11 +550,11 @@ GitHost <- R6::R6Class( httr2::req_headers("Authorization" = paste0("Bearer ", token)) |> httr2::req_perform(), silent = TRUE) if (!is.null(response)) { - private$check_token_scopes(response, token) - TRUE + check <- private$check_token_scopes(response, token) } else { - FALSE + check <- FALSE } + return(check) }, # Helper to extract organizations and repositories from vector of full names diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 916892ed..6dc58083 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -29,8 +29,12 @@ GitHostGitHub <- R6::R6Class( # Default token name token_name = "GITHUB_PAT", + # Minimum access scopes for token + min_access_scopes = c("public_repo", "read:org", "read:user"), + # Access scopes for token - access_scopes = c("public_repo", "read:org", "read:user"), + access_scopes = c("public_repo", "read:org", "read:user", + "repo", "admin:org", "user"), # Methods for engines engine_methods = list( @@ -124,10 +128,10 @@ GitHostGitHub <- R6::R6Class( # Check token scopes # token parameter only for need of super method check_token_scopes = function(response, token = NULL) { - token_scopes <- response$headers$`x-oauth-scopes` %>% + private$token_scopes <- response$headers$`x-oauth-scopes` %>% stringr::str_split(", ") %>% unlist() - all(private$access_scopes %in% token_scopes) + all(private$access_scopes %in% private$token_scopes) }, # Retrieve only important info from repositories response diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index 7b702034..6c1d2b5d 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -84,6 +84,9 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", # Default token name token_name = "GITLAB_PAT", + # Minimum access scopes for token + min_access_scopes = c("read_api"), + # Access scopes for token access_scopes = c("api", "read_api"), @@ -184,21 +187,24 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", # check token scopes # response parameter only for need of super method check_token_scopes = function(response = NULL, token) { - token_scopes <- httr2::request(private$endpoints$tokens) %>% - httr2::req_headers("Authorization" = paste0("Bearer ", token)) %>% - httr2::req_perform() %>% - httr2::resp_body_json() %>% - purrr::keep(~ .$active) %>% - purrr::map(function(pat) { - data.frame(scopes = unlist(pat$scopes), date = pat$last_used_at) - }) %>% - purrr::list_rbind() %>% - dplyr::filter( - date == max(date) - ) %>% - dplyr::select(scopes) %>% - unlist() - any(private$access_scopes %in% token_scopes) + private$token_scopes <- try({ + httr2::request(private$endpoints$tokens) %>% + httr2::req_headers("Authorization" = paste0("Bearer ", token)) %>% + httr2::req_perform() %>% + httr2::resp_body_json() %>% + purrr::keep(~ .$active) %>% + purrr::map(function(pat) { + data.frame(scopes = unlist(pat$scopes), date = pat$last_used_at) + }) %>% + purrr::list_rbind() %>% + dplyr::filter( + date == max(date) + ) %>% + dplyr::select(scopes) %>% + unlist() + }, + silent = TRUE) + any(private$access_scopes %in% private$token_scopes) }, # Retrieve only important info from repositories response diff --git a/inst/set_tokens.R b/inst/set_tokens.R new file mode 100644 index 00000000..61db699e --- /dev/null +++ b/inst/set_tokens.R @@ -0,0 +1,26 @@ +# to TEST create tokens with insufficient scopes + +create_gitstats() |> + set_github_host( + orgs = "r-world-devs", + token = Sys.getenv("GITHUB_PAT_INSUFFICIENT") + ) |> + get_repos() + +create_gitstats() |> + set_gitlab_host( + orgs = "mbtests" + ) |> + get_repos() + +create_gitstats() |> + set_gitlab_host( + orgs = "mbtests", + token = Sys.getenv("GITLAB_PAT_PUBLIC_INSUFFICIENT") + ) |> + get_repos() + +create_gitstats() |> + set_github_host( + orgs = "r-world-devs" + ) From 6daf6a947bca1e06816903b9284c67b4e0365fc8 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 14 Oct 2024 11:21:49 +0000 Subject: [PATCH 33/94] Skip and remove superfluous. --- tests/testthat/test-helpers.R | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 4240b22c..2cda9185 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -1,8 +1,3 @@ -test_that("`get_group_id()` gets group's id", { - gl_group_id <- test_rest_gitlab_priv$get_group_id("mbtests") - expect_equal(gl_group_id, 63684059) -}) - test_that("`set_searching_scope` does not throw error when `orgs` or `repos` are defined", { expect_snapshot( gitlab_testhost_priv$set_searching_scope(orgs = "mbtests", repos = NULL, verbose = TRUE) @@ -128,6 +123,7 @@ test_that("`set_default_token` sets default token for public GitHub", { }) test_that("`test_token` works properly", { + skip_on_cran() expect_true( github_testhost_priv$test_token(Sys.getenv("GITHUB_PAT")) ) From 3b3072b9dc359500221ec09e8f2a1aab82902aed Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 14 Oct 2024 12:15:08 +0000 Subject: [PATCH 34/94] Fix token check. The logic in `set_default_token` was wrong: if all possible tokens failed in for loop, no error was returned if last check token was not empty (but still invalid). This invalid token was passed to return (as it passed non-empty condition), with no message to user, but passing token to GitHost. Also, the check in GitHub `check_token_scopes` was wrong, and it is fixed now. --- R/GitHost.R | 30 +++++++++++++++--------------- R/GitHostGitHub.R | 12 +++++++++--- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index 04b7e0c5..80183a5b 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -521,24 +521,24 @@ GitHost <- R6::R6Class( } else { pat_names <- names(Sys.getenv()[grepl(primary_token_name, names(Sys.getenv()))]) possible_tokens <- pat_names[pat_names != primary_token_name] - for (token_name in possible_tokens) { - if (private$test_token(Sys.getenv(token_name))) { - token <- Sys.getenv(token_name) - if (verbose) { - cli::cli_alert_info("Using PAT from {token_name} envar.") - } - break + token_checks <- purrr::map_lgl(possible_tokens, function(token_name) { + private$test_token(Sys.getenv(token_name)) + }) + if (!any(token_checks)) { + cli::cli_abort(c( + "x" = "No sufficient token found among: [{paste0(pat_names, collapse = ', ')}].", + "i" = "Check if you have correct token.", + "!" = "Scope that is needed: [{paste0(private$min_access_scopes, collapse = ', ')}]." + ), + call = NULL) + } else { + token_name <- possible_tokens[token_checks][1] + if (verbose) { + cli::cli_alert_info("Using PAT from {token_name} envar.") } + token <- Sys.getenv(token_name) } } - if (token == "") { - cli::cli_abort(c( - "x" = "No sufficient token found among: [{paste0(pat_names, collapse = ', ')}].", - "i" = "Check if you have correct token.", - "!" = "Scope that is needed: [{paste0(private$min_access_scopes, collapse = ', ')}]." - ), - call = NULL) - } return(token) }, diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 6dc58083..b258e9c5 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -33,8 +33,11 @@ GitHostGitHub <- R6::R6Class( min_access_scopes = c("public_repo", "read:org", "read:user"), # Access scopes for token - access_scopes = c("public_repo", "read:org", "read:user", - "repo", "admin:org", "user"), + access_scopes = list( + org = c("read:org", "admin:org"), + repo = c("public_repo", "repo"), + user = c("read:user", "user") + ), # Methods for engines engine_methods = list( @@ -131,7 +134,10 @@ GitHostGitHub <- R6::R6Class( private$token_scopes <- response$headers$`x-oauth-scopes` %>% stringr::str_split(", ") %>% unlist() - all(private$access_scopes %in% private$token_scopes) + org_scopes <- any(private$access_scopes$org %in% private$token_scopes) + repo_scopes <- any(private$access_scopes$repo %in% private$token_scopes) + user_scopes <- any(private$access_scopes$user %in% private$token_scopes) + all(c(org_scopes, repo_scopes, user_scopes)) }, # Retrieve only important info from repositories response From 7bfdf65761a489c4f724fab88fe6303217902dbd Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 14 Oct 2024 12:59:21 +0000 Subject: [PATCH 35/94] Fix when host is wrong. Check for wrong host was moved earlier, when tokens are tested. --- R/GitHost.R | 21 ++++++++++++--------- tests/testthat/_snaps/helpers.md | 3 ++- tests/testthat/_snaps/set_host.md | 10 +++------- tests/testthat/test-set_host.R | 2 -- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index 80183a5b..09725a90 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -482,12 +482,6 @@ GitHost <- R6::R6Class( private$engines$rest$response(endpoint = endpoint) }, error = function(e) { - if (!is.null(e$parent$message) && grepl("Could not resolve host", e$parent$message)) { - cli::cli_abort( - cli::col_red(e$parent$message), - call = NULL - ) - } if (grepl("404", e)) { cli::cli_abort( c( @@ -546,9 +540,18 @@ GitHost <- R6::R6Class( test_token = function(token) { response <- NULL test_endpoint <- private$test_endpoint - try(response <- httr2::request(test_endpoint) |> - httr2::req_headers("Authorization" = paste0("Bearer ", token)) |> - httr2::req_perform(), silent = TRUE) + response <- tryCatch({ + httr2::request(test_endpoint) |> + httr2::req_headers("Authorization" = paste0("Bearer ", token)) |> + httr2::req_perform() + }, + error = function(e) { + if (!is.null(e$parent) && grepl("Could not resolve host", e$parent$message)) { + cli::cli_abort(e$parent$message, call = NULL) + } else { + NULL + } + }) if (!is.null(response)) { check <- private$check_token_scopes(response, token) } else { diff --git a/tests/testthat/_snaps/helpers.md b/tests/testthat/_snaps/helpers.md index 84602e49..aec3df67 100644 --- a/tests/testthat/_snaps/helpers.md +++ b/tests/testthat/_snaps/helpers.md @@ -25,7 +25,8 @@ # `check_token()` prints error when token exists but does not grant access x Token exists but does not grant access. - i Check if you use correct token. Check scopes your token is using. + i Check if you use correct token. + ! Scope that is needed: [public_repo, read:org, read:user]. # check_endpoint returns error if they are not correct diff --git a/tests/testthat/_snaps/set_host.md b/tests/testthat/_snaps/set_host.md index c459840b..e3e8c181 100644 --- a/tests/testthat/_snaps/set_host.md +++ b/tests/testthat/_snaps/set_host.md @@ -77,20 +77,16 @@ # Error shows, when wrong input is passed when setting connection and host is not passed x Token exists but does not grant access. - i Check if you use correct token. Check scopes your token is using. + i Check if you use correct token. + ! Scope that is needed: [read_api]. --- Code create_gitstats() %>% set_github_host(host = "wrong.url", orgs = c("openpharma", "r_world_devs")) - Message - i Searching scope set to [org]. - i Checking organizations... Condition - Error in `purrr::map()`: - i In index: 1. - Caused by error: + Error: ! Could not resolve host: wrong.url # Error pops out, when two clients of the same url api are passed as input diff --git a/tests/testthat/test-set_host.R b/tests/testthat/test-set_host.R index f0ddb67f..0aaba037 100644 --- a/tests/testthat/test-set_host.R +++ b/tests/testthat/test-set_host.R @@ -98,7 +98,6 @@ test_that("Error shows if organizations are not specified and host is not passed test_that("Error shows, when wrong input is passed when setting connection and host is not passed", { test_gitstats <- create_gitstats() - expect_snapshot_error( set_gitlab_host( gitstats_object = test_gitstats, @@ -106,7 +105,6 @@ test_that("Error shows, when wrong input is passed when setting connection and h token = Sys.getenv("GITLAB_PAT_PUBLIC") ) ) - expect_snapshot({ create_gitstats() %>% set_github_host( From 83ae94f18cb7655db3b38723b47379477f5eaf78 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 14 Oct 2024 14:16:23 +0000 Subject: [PATCH 36/94] Try fix test after fixing check. --- tests/testthat/test-helpers.R | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 2cda9185..e6789629 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -55,6 +55,11 @@ test_that("when token is proper token is passed", { token = Sys.getenv("GITHUB_PAT"), mode = "private" ) + mockery::stub( + github_testhost_priv$check_token, + "private$test_token", + TRUE + ) expect_equal( github_testhost_priv$check_token(Sys.getenv("GITHUB_PAT")), Sys.getenv("GITHUB_PAT") From c470433da6d390319d87c816ffcf40513949f2f6 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 14 Oct 2024 14:17:15 +0000 Subject: [PATCH 37/94] Lint. --- R/GitHost.R | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index 09725a90..2975c1a2 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -544,14 +544,14 @@ GitHost <- R6::R6Class( httr2::request(test_endpoint) |> httr2::req_headers("Authorization" = paste0("Bearer ", token)) |> httr2::req_perform() - }, - error = function(e) { - if (!is.null(e$parent) && grepl("Could not resolve host", e$parent$message)) { - cli::cli_abort(e$parent$message, call = NULL) - } else { - NULL - } - }) + }, + error = function(e) { + if (!is.null(e$parent) && grepl("Could not resolve host", e$parent$message)) { + cli::cli_abort(e$parent$message, call = NULL) + } else { + NULL + } + }) if (!is.null(response)) { check <- private$check_token_scopes(response, token) } else { From 3a26b74d47d1525599a27fbfb4d3d05f827164f5 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 14 Oct 2024 14:28:35 +0000 Subject: [PATCH 38/94] Try fix tests on GHA. --- tests/testthat/test-helpers.R | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index e6789629..998b0f26 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -111,6 +111,11 @@ test_that("`check_if_public` works correctly", { }) test_that("`set_default_token` sets default token for public GitHub", { + stub::mockery( + github_testhost_priv$set_default_token, + "private$test_token", + TRUE + ) expect_snapshot( default_token <- github_testhost_priv$set_default_token( verbose = TRUE From 56360528cbdcee24be0726089069b80c26b4ceb0 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 14 Oct 2024 14:37:13 +0000 Subject: [PATCH 39/94] Typo. --- tests/testthat/test-helpers.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 998b0f26..888bb33b 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -111,7 +111,7 @@ test_that("`check_if_public` works correctly", { }) test_that("`set_default_token` sets default token for public GitHub", { - stub::mockery( + mockery::stub( github_testhost_priv$set_default_token, "private$test_token", TRUE From accef8a934c7a3ab7f44c58fb9e0a69ecb8fa261 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 14 Oct 2024 14:43:33 +0000 Subject: [PATCH 40/94] Try fix test on GHA. --- tests/testthat/test-helpers.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 888bb33b..a134cbb5 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -135,7 +135,7 @@ test_that("`set_default_token` sets default token for public GitHub", { test_that("`test_token` works properly", { skip_on_cran() expect_true( - github_testhost_priv$test_token(Sys.getenv("GITHUB_PAT")) + github_testhost_priv$test_token(Sys.getenv("TEST_GITHUB_PAT")) ) expect_false( github_testhost_priv$test_token("false_token") From eda6be3d63b5239b3567aa46a985f792d090a15c Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 14 Oct 2024 15:15:10 +0000 Subject: [PATCH 41/94] Fix job, try use tailored Github Pat. --- .github/workflows/test-coverage.yaml | 2 +- tests/testthat/test-helpers.R | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 648d9a22..f3419212 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -10,7 +10,7 @@ jobs: test-coverage: runs-on: ubuntu-latest env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PAT: ${{ secrets.TEST_GITHUB_PAT }} GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT}} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} USE_RENV: "FALSE" diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index a134cbb5..888bb33b 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -135,7 +135,7 @@ test_that("`set_default_token` sets default token for public GitHub", { test_that("`test_token` works properly", { skip_on_cran() expect_true( - github_testhost_priv$test_token(Sys.getenv("TEST_GITHUB_PAT")) + github_testhost_priv$test_token(Sys.getenv("GITHUB_PAT")) ) expect_false( github_testhost_priv$test_token("false_token") From 31593179c5100201cee961362e0a6cf596bdb071 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 15 Oct 2024 07:13:38 +0000 Subject: [PATCH 42/94] Try fix checks (building vignettes). For checks, use token (set up as GHA secret) with wider scope than the default one on GHA. --- .github/workflows/R-CMD-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 95e4e92b..821821ad 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -23,7 +23,7 @@ jobs: - {os: ubuntu-latest, r: 'release'} env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PAT: ${{ secrets.TEST_GITHUB_TOKEN }} GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT}} R_KEEP_PKG_SOURCE: yes USE_RENV: "FALSE" From f56d8d127984b22742b60b594f6ae4bddfbf923f Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 15 Oct 2024 07:14:05 +0000 Subject: [PATCH 43/94] Try use new token set up on GHA with wider scope. --- tests/testthat/test-helpers.R | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 888bb33b..64b49f07 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -111,11 +111,7 @@ test_that("`check_if_public` works correctly", { }) test_that("`set_default_token` sets default token for public GitHub", { - mockery::stub( - github_testhost_priv$set_default_token, - "private$test_token", - TRUE - ) + skip_on_cran() expect_snapshot( default_token <- github_testhost_priv$set_default_token( verbose = TRUE From d8a771df5dc9b1f73cb578ecba7fa0cbca4822de Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 15 Oct 2024 07:22:09 +0000 Subject: [PATCH 44/94] Fix typo. --- .github/workflows/R-CMD-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 821821ad..76d28d4f 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -23,7 +23,7 @@ jobs: - {os: ubuntu-latest, r: 'release'} env: - GITHUB_PAT: ${{ secrets.TEST_GITHUB_TOKEN }} + GITHUB_PAT: ${{ secrets.TEST_GITHUB_PAT}} GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT}} R_KEEP_PKG_SOURCE: yes USE_RENV: "FALSE" From c9703ec49d0769f60b8619616ec3ae7b616ef028 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 15 Oct 2024 07:51:14 +0000 Subject: [PATCH 45/94] Test-cover case when there are no default tokens. --- tests/testthat/_snaps/helpers.md | 10 ++++++++++ tests/testthat/test-helpers.R | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/tests/testthat/_snaps/helpers.md b/tests/testthat/_snaps/helpers.md index aec3df67..bfc9ec67 100644 --- a/tests/testthat/_snaps/helpers.md +++ b/tests/testthat/_snaps/helpers.md @@ -41,6 +41,16 @@ Message i Using PAT from GITHUB_PAT envar. +# `set_default_token` returns error if none are found + + Code + github_testhost_priv$set_default_token(verbose = TRUE) + Condition + Error: + x No sufficient token found among: [GITHUB_PAT, GITHUB_PAT_INSUFFICIENT, GITHUB_PAT_MIN, GITHUB_PAT_ROCHE, TEST_GITHUB_PAT]. + i Check if you have correct token. + ! Scope that is needed: [public_repo, read:org, read:user]. + # `set_default_token` sets default token for GitLab Code diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 64b49f07..0c986021 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -128,6 +128,19 @@ test_that("`set_default_token` sets default token for public GitHub", { ) }) +test_that("`set_default_token` returns error if none are found", { + mockery::stub( + github_testhost_priv$set_default_token, + "private$test_token", + FALSE + ) + expect_snapshot({ + github_testhost_priv$set_default_token( + verbose = TRUE + ) + }, error = TRUE) +}) + test_that("`test_token` works properly", { skip_on_cran() expect_true( From f53947e77ba1ae057c02814a49229b285cffacfe Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 15 Oct 2024 07:57:14 +0000 Subject: [PATCH 46/94] Remove snapshot, as on GHA different possible tokens are shown. --- tests/testthat/_snaps/helpers.md | 10 ---------- tests/testthat/test-helpers.R | 6 +++--- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/tests/testthat/_snaps/helpers.md b/tests/testthat/_snaps/helpers.md index bfc9ec67..aec3df67 100644 --- a/tests/testthat/_snaps/helpers.md +++ b/tests/testthat/_snaps/helpers.md @@ -41,16 +41,6 @@ Message i Using PAT from GITHUB_PAT envar. -# `set_default_token` returns error if none are found - - Code - github_testhost_priv$set_default_token(verbose = TRUE) - Condition - Error: - x No sufficient token found among: [GITHUB_PAT, GITHUB_PAT_INSUFFICIENT, GITHUB_PAT_MIN, GITHUB_PAT_ROCHE, TEST_GITHUB_PAT]. - i Check if you have correct token. - ! Scope that is needed: [public_repo, read:org, read:user]. - # `set_default_token` sets default token for GitLab Code diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 0c986021..34c498bb 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -134,11 +134,11 @@ test_that("`set_default_token` returns error if none are found", { "private$test_token", FALSE ) - expect_snapshot({ + expect_error({ github_testhost_priv$set_default_token( - verbose = TRUE + verbose = FALSE ) - }, error = TRUE) + }) }) test_that("`test_token` works properly", { From 405ee3c1601c8482df6e6721ffbce3aa0d115d1d Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 15 Oct 2024 11:49:27 +0000 Subject: [PATCH 47/94] Move parsing methods for repositories to lower-level API clients. --- R/EngineGraphQLGitHub.R | 34 ++++++++++++ R/EngineGraphQLGitLab.R | 38 +++++++++++++ R/EngineRest.R | 26 +++++++++ R/EngineRestGitHub.R | 25 +++++++++ R/EngineRestGitLab.R | 21 +++++++ R/GitHost.R | 36 ++---------- R/GitHostGitHub.R | 58 -------------------- R/GitHostGitLab.R | 58 -------------------- R/GitStats.R | 4 +- tests/testthat/_snaps/01-get_repos-GitHub.md | 4 +- tests/testthat/_snaps/01-get_repos-GitLab.md | 6 +- tests/testthat/test-01-get_repos-GitHub.R | 8 +-- tests/testthat/test-01-get_repos-GitLab.R | 8 +-- 13 files changed, 164 insertions(+), 162 deletions(-) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index ed03264f..966a567c 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -71,6 +71,40 @@ EngineGraphQLGitHub <- R6::R6Class( return(full_repos_list) }, + # Parses repositories list into table. + prepare_repos_table = function(repos_list) { + if (length(repos_list) > 0) { + repos_table <- purrr::map(repos_list, function(repo) { + repo$default_branch <- if (!is.null(repo$default_branch)) { + repo$default_branch$name + } else { + "" + } + last_activity_at <- as.POSIXct(repo$last_activity_at) + if (length(last_activity_at) == 0) { + last_activity_at <- gts_to_posixt(repo$created_at) + } + repo$languages <- purrr::map_chr(repo$languages$nodes, ~ .$name) %>% + paste0(collapse = ", ") + repo$created_at <- gts_to_posixt(repo$created_at) + repo$issues_open <- repo$issues_open$totalCount + repo$issues_closed <- repo$issues_closed$totalCount + repo$last_activity_at <- last_activity_at + repo$organization <- repo$organization$login + repo <- data.frame(repo) %>% + dplyr::relocate( + default_branch, + .after = repo_name + ) + return(repo) + }) %>% + purrr::list_rbind() + } else { + repos_table <- NULL + } + return(repos_table) + }, + # Iterator over pulling commits from all repositories. get_commits_from_repos = function(org, repos_names, diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index ccf47280..9b826d3a 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -77,6 +77,44 @@ EngineGraphQLGitLab <- R6::R6Class( return(full_repos_list) }, + # Parses repositories list into table. + prepare_repos_table = function(repos_list) { + if (length(repos_list) > 0) { + repos_table <- purrr::map(repos_list, function(repo) { + repo <- repo$node + repo$default_branch <- repo$repository$rootRef %||% "" + repo$repository <- NULL + repo$languages <- if (length(repo$languages) > 0) { + purrr::map_chr(repo$languages, ~ .$name) %>% + paste0(collapse = ", ") + } else { + "" + } + repo$created_at <- gts_to_posixt(repo$created_at) + repo$issues_open <- repo$issues$opened + repo$issues_closed <- repo$issues$closed + repo$issues <- NULL + repo$last_activity_at <- as.POSIXct(repo$last_activity_at) + repo$organization <- repo$namespace$path + repo$namespace <- NULL + repo$repo_path <- NULL # temporary to close issue 338 + data.frame(repo) + }) %>% + purrr::list_rbind() %>% + dplyr::relocate( + repo_url, + .after = organization + ) %>% + dplyr::relocate( + default_branch, + .after = repo_name + ) + } else { + repos_table <- NULL + } + return(repos_table) + }, + # Pull all given files from all repositories of a group. # This is a one query way to get all the necessary info. # However it may fail if query is too complex (too many files in file_paths). diff --git a/R/EngineRest.R b/R/EngineRest.R index d20ab0a5..6b8ad1fc 100644 --- a/R/EngineRest.R +++ b/R/EngineRest.R @@ -36,6 +36,32 @@ EngineRest <- R6::R6Class("EngineRest", result <- list() } return(result) + }, + + # Prepare table for repositories content + prepare_repos_table = function(repos_list, verbose = TRUE) { + repos_dt <- purrr::map(repos_list, function(repo) { + repo <- purrr::map(repo, function(attr) { + attr <- attr %||% "" + }) + data.frame(repo) + }) %>% + purrr::list_rbind() + if (verbose) { + cli::cli_alert_info("Preparing repositories table...") + } + if (length(repos_dt) > 0) { + repos_dt <- dplyr::mutate( + repos_dt, + repo_id = as.character(repo_id), + created_at = as.POSIXct(created_at), + last_activity_at = as.POSIXct(last_activity_at), + forks = as.integer(forks), + issues_open = as.integer(issues_open), + issues_closed = as.integer(issues_closed) + ) + } + return(repos_dt) } ), diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index 694a688b..7ddd1784 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -67,6 +67,31 @@ EngineRestGitHub <- R6::R6Class( return(search_output) }, + # Retrieve only important info from repositories response + tailor_repos_response = function(repos_response) { + repos_list <- purrr::map(repos_response, function(repo) { + list( + "repo_id" = repo$id, + "repo_name" = repo$name, + "default_branch" = repo$default_branch, + "stars" = repo$stargazers_count, + "forks" = repo$forks_count, + "created_at" = gts_to_posixt(repo$created_at), + "last_activity_at" = if (!is.null(repo$pushed_at)) { + gts_to_posixt(repo$pushed_at) + } else { + gts_to_posixt(repo$created_at) + }, + "languages" = repo$language, + "issues_open" = repo$issues_open, + "issues_closed" = repo$issues_closed, + "organization" = repo$owner$login, + "repo_url" = repo$html_url + ) + }) + return(repos_list) + }, + #' Pull all repositories URLS from organization get_repos_urls = function(type, org) { repos_urls <- self$response( diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 6820a147..70683fb2 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -66,6 +66,27 @@ EngineRestGitLab <- R6::R6Class( return(search_output) }, + # Retrieve only important info from repositories response + tailor_repos_response = function(repos_response) { + repos_list <- purrr::map(repos_response, function(project) { + list( + "repo_id" = project$id, + "repo_name" = project$name, + "default_branch" = project$default_branch, + "stars" = project$star_count, + "forks" = project$fork_count, + "created_at" = project$created_at, + "last_activity_at" = project$last_activity_at, + "languages" = paste0(project$languages, collapse = ", "), + "issues_open" = project$issues_open, + "issues_closed" = project$issues_closed, + "organization" = project$namespace$path, + "repo_url" = project$web_url + ) + }) + return(repos_list) + }, + # Pull all repositories URLs from organization get_repos_urls = function(type, org) { repos_urls <- self$response( diff --git a/R/GitHost.R b/R/GitHost.R index 2975c1a2..f568b8bf 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -626,7 +626,7 @@ GitHost <- R6::R6Class( org = org, type = type ) %>% - private$prepare_repos_table_from_graphql() + graphql_engine$prepare_repos_table() if (!is.null(repos)) { repos_table <- repos_table %>% dplyr::filter(repo_name %in% repos) @@ -725,8 +725,8 @@ GitHost <- R6::R6Class( if (!raw_output) { rest_engine <- private$engines$rest repos_table <- repos_response %>% - private$tailor_repos_response() %>% - private$prepare_repos_table_from_rest( + rest_engine$tailor_repos_response() %>% + rest_engine$prepare_repos_table( verbose = verbose ) %>% rest_engine$get_repos_issues( @@ -767,8 +767,8 @@ GitHost <- R6::R6Class( if (!raw_output) { rest_engine <- private$engines$rest repos_table <- repos_response %>% - private$tailor_repos_response() %>% - private$prepare_repos_table_from_rest( + rest_engine$tailor_repos_response() %>% + rest_engine$prepare_repos_table( verbose = verbose ) %>% rest_engine$get_repos_issues( @@ -842,32 +842,6 @@ GitHost <- R6::R6Class( } }, - # Prepare table for repositories content - prepare_repos_table_from_rest = function(repos_list, verbose = TRUE) { - repos_dt <- purrr::map(repos_list, function(repo) { - repo <- purrr::map(repo, function(attr) { - attr <- attr %||% "" - }) - data.frame(repo) - }) %>% - purrr::list_rbind() - if (verbose) { - cli::cli_alert_info("Preparing repositories table...") - } - if (length(repos_dt) > 0) { - repos_dt <- dplyr::mutate( - repos_dt, - repo_id = as.character(repo_id), - created_at = as.POSIXct(created_at), - last_activity_at = as.POSIXct(last_activity_at), - forks = as.integer(forks), - issues_open = as.integer(issues_open), - issues_closed = as.integer(issues_closed) - ) - } - return(repos_dt) - }, - # Pull files content from organizations get_files_content_from_orgs = function(file_path, host_files_structure = NULL, diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index b258e9c5..7f0a6c3c 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -140,64 +140,6 @@ GitHostGitHub <- R6::R6Class( all(c(org_scopes, repo_scopes, user_scopes)) }, - # Retrieve only important info from repositories response - tailor_repos_response = function(repos_response) { - repos_list <- purrr::map(repos_response, function(repo) { - list( - "repo_id" = repo$id, - "repo_name" = repo$name, - "default_branch" = repo$default_branch, - "stars" = repo$stargazers_count, - "forks" = repo$forks_count, - "created_at" = gts_to_posixt(repo$created_at), - "last_activity_at" = if (!is.null(repo$pushed_at)) { - gts_to_posixt(repo$pushed_at) - } else { - gts_to_posixt(repo$created_at) - }, - "languages" = repo$language, - "issues_open" = repo$issues_open, - "issues_closed" = repo$issues_closed, - "organization" = repo$owner$login, - "repo_url" = repo$html_url - ) - }) - return(repos_list) - }, - - # Parses repositories list into table. - prepare_repos_table_from_graphql = function(repos_list) { - if (length(repos_list) > 0) { - repos_table <- purrr::map_dfr(repos_list, function(repo) { - repo$default_branch <- if (!is.null(repo$default_branch)) { - repo$default_branch$name - } else { - "" - } - last_activity_at <- as.POSIXct(repo$last_activity_at) - if (length(last_activity_at) == 0) { - last_activity_at <- gts_to_posixt(repo$created_at) - } - repo$languages <- purrr::map_chr(repo$languages$nodes, ~ .$name) %>% - paste0(collapse = ", ") - repo$created_at <- gts_to_posixt(repo$created_at) - repo$issues_open <- repo$issues_open$totalCount - repo$issues_closed <- repo$issues_closed$totalCount - repo$last_activity_at <- last_activity_at - repo$organization <- repo$organization$login - repo <- data.frame(repo) %>% - dplyr::relocate( - default_branch, - .after = repo_name - ) - return(repo) - }) - } else { - repos_table <- NULL - } - return(repos_table) - }, - # Add `api_url` column to table. add_repo_api_url = function(repos_table) { if (!is.null(repos_table) && nrow(repos_table) > 0) { diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index 6c1d2b5d..281fe017 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -207,64 +207,6 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", any(private$access_scopes %in% private$token_scopes) }, - # Retrieve only important info from repositories response - tailor_repos_response = function(repos_response) { - repos_list <- purrr::map(repos_response, function(project) { - list( - "repo_id" = project$id, - "repo_name" = project$name, - "default_branch" = project$default_branch, - "stars" = project$star_count, - "forks" = project$fork_count, - "created_at" = project$created_at, - "last_activity_at" = project$last_activity_at, - "languages" = paste0(project$languages, collapse = ", "), - "issues_open" = project$issues_open, - "issues_closed" = project$issues_closed, - "organization" = project$namespace$path, - "repo_url" = project$web_url - ) - }) - return(repos_list) - }, - - # Parses repositories list into table. - prepare_repos_table_from_graphql = function(repos_list) { - if (length(repos_list) > 0) { - repos_table <- purrr::map_dfr(repos_list, function(repo) { - repo <- repo$node - repo$default_branch <- repo$repository$rootRef %||% "" - repo$repository <- NULL - repo$languages <- if (length(repo$languages) > 0) { - purrr::map_chr(repo$languages, ~ .$name) %>% - paste0(collapse = ", ") - } else { - "" - } - repo$created_at <- gts_to_posixt(repo$created_at) - repo$issues_open <- repo$issues$opened - repo$issues_closed <- repo$issues$closed - repo$issues <- NULL - repo$last_activity_at <- as.POSIXct(repo$last_activity_at) - repo$organization <- repo$namespace$path - repo$namespace <- NULL - repo$repo_path <- NULL # temporary to close issue 338 - data.frame(repo) - }) %>% - dplyr::relocate( - repo_url, - .after = organization - ) %>% - dplyr::relocate( - default_branch, - .after = repo_name - ) - } else { - repos_table <- NULL - } - return(repos_table) - }, - # Add `api_url` column to table. add_repo_api_url = function(repos_table) { if (!is.null(repos_table) && nrow(repos_table) > 0) { diff --git a/R/GitStats.R b/R/GitStats.R index 96071593..3e75eb60 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -709,7 +709,8 @@ GitStats <- R6::R6Class( } }) %>% purrr::list_rbind() %>% - private$add_stats_to_repos() + private$add_stats_to_repos() %>% + dplyr::as_tibble() return(repos_table) }, @@ -974,7 +975,6 @@ GitStats <- R6::R6Class( dplyr::mutate( package_usage = ifelse(api_url %in% duplicated_repos, "import, library", package_usage) ) - rownames(package_usage_table) <- seq_len(nrow(package_usage_table)) } return(package_usage_table) }, diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index 0c71258f..72bd89bc 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -8,8 +8,8 @@ # `prepare_repos_table()` prepares repos table Code - gh_repos_by_code_table <- github_testhost_priv$prepare_repos_table_from_rest( - repos_list = test_mocker$use("gh_repos_by_code_tailored")) + gh_repos_by_code_table <- test_rest_github$prepare_repos_table(repos_list = test_mocker$ + use("gh_repos_by_code_tailored")) Message i Preparing repositories table... diff --git a/tests/testthat/_snaps/01-get_repos-GitLab.md b/tests/testthat/_snaps/01-get_repos-GitLab.md index c6dab9da..b1db1759 100644 --- a/tests/testthat/_snaps/01-get_repos-GitLab.md +++ b/tests/testthat/_snaps/01-get_repos-GitLab.md @@ -5,11 +5,11 @@ Output [1] "\n query GetReposByOrg($org: ID! $repo_cursor: String!) {\n group(fullPath: $org) {\n projects(first: 100 after: $repo_cursor) {\n \n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n namespace {\n path\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" -# GitHost prepares table from GitLab repositories response +# REST client prepares table from GitLab repositories response Code - gl_repos_by_code_table <- gitlab_testhost_priv$prepare_repos_table_from_rest( - repos_list = test_mocker$use("gl_repos_by_code_tailored")) + gl_repos_by_code_table <- test_rest_gitlab$prepare_repos_table(repos_list = test_mocker$ + use("gl_repos_by_code_tailored")) Message i Preparing repositories table... diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 3c09ab50..c4bc6da6 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -183,7 +183,7 @@ test_that("`get_repos_by_code()` for GitHub prepares a raw (raw_output = TRUE) s test_that("GitHub tailors precisely `repos_list`", { gh_repos_by_code <- test_mocker$use("gh_repos_by_code") gh_repos_by_code_tailored <- - github_testhost_priv$tailor_repos_response(gh_repos_by_code) + test_rest_github$tailor_repos_response(gh_repos_by_code) gh_repos_by_code_tailored %>% expect_type("list") %>% expect_length(length(gh_repos_by_code)) @@ -204,7 +204,7 @@ test_that("GitHub tailors precisely `repos_list`", { test_that("`prepare_repos_table()` prepares repos table", { expect_snapshot( - gh_repos_by_code_table <- github_testhost_priv$prepare_repos_table_from_rest( + gh_repos_by_code_table <- test_rest_github$prepare_repos_table( repos_list = test_mocker$use("gh_repos_by_code_tailored") ) ) @@ -258,7 +258,7 @@ test_that("`get_repos_with_code_from_orgs()` works", { }) test_that("GitHub prepares repos table from repositories response", { - gh_repos_table <- github_testhost_priv$prepare_repos_table_from_graphql( + gh_repos_table <- test_graphql_github$prepare_repos_table( repos_list = test_mocker$use("gh_repos_from_org") ) expect_repos_table( @@ -277,7 +277,7 @@ test_that("GitHost adds `repo_api_url` column to GitHub repos table", { test_that("`get_all_repos()` works as expected", { mockery::stub( github_testhost_priv$get_all_repos, - "private$prepare_repos_table_from_graphql", + "graphql_engine$prepare_repos_table", test_mocker$use("gh_repos_table_with_api_url") ) expect_snapshot( diff --git a/tests/testthat/test-01-get_repos-GitLab.R b/tests/testthat/test-01-get_repos-GitLab.R index 0fbe209f..751455f6 100644 --- a/tests/testthat/test-01-get_repos-GitLab.R +++ b/tests/testthat/test-01-get_repos-GitLab.R @@ -149,7 +149,7 @@ test_that("`get_repos_languages` works", { }) test_that("`prepare_repos_table()` prepares repos table", { - gl_repos_table <- gitlab_testhost_priv$prepare_repos_table_from_graphql( + gl_repos_table <- test_graphql_gitlab$prepare_repos_table( repos_list = test_mocker$use("gl_repos_from_org") ) expect_repos_table( @@ -168,7 +168,7 @@ test_that("GitHost adds `repo_api_url` column to GitLab repos table", { test_that("`tailor_repos_response()` tailors precisely `repos_list`", { gl_repos_by_code <- test_mocker$use("gl_search_repos_by_code") gl_repos_by_code_tailored <- - gitlab_testhost_priv$tailor_repos_response(gl_repos_by_code) + test_rest_gitlab$tailor_repos_response(gl_repos_by_code) gl_repos_by_code_tailored %>% expect_type("list") %>% expect_length(length(gl_repos_by_code)) @@ -188,9 +188,9 @@ test_that("`tailor_repos_response()` tailors precisely `repos_list`", { test_mocker$cache(gl_repos_by_code_tailored) }) -test_that("GitHost prepares table from GitLab repositories response", { +test_that("REST client prepares table from GitLab repositories response", { expect_snapshot( - gl_repos_by_code_table <- gitlab_testhost_priv$prepare_repos_table_from_rest( + gl_repos_by_code_table <- test_rest_gitlab$prepare_repos_table( repos_list = test_mocker$use("gl_repos_by_code_tailored") ) ) From 2c143d83181acbdf6085ec5959c1b8eedeff892a Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 15 Oct 2024 12:21:29 +0000 Subject: [PATCH 48/94] Move parsing methods for commits, files, users and release logs. --- R/EngineGraphQLGitHub.R | 127 +++++++++++++ R/EngineGraphQLGitLab.R | 110 +++++++++++ R/EngineRestGitLab.R | 62 ++++++ R/GitHost.R | 10 +- R/GitHostGitHub.R | 129 +------------ R/GitHostGitLab.R | 178 +----------------- R/GitStats.R | 12 +- tests/testthat/test-get_commits-GitHub.R | 4 +- tests/testthat/test-get_commits-GitLab.R | 4 +- .../testthat/test-get_files_content-GitHub.R | 6 +- .../testthat/test-get_files_content-GitLab.R | 10 +- tests/testthat/test-get_release-GitHub.R | 4 +- tests/testthat/test-get_release-GitLab.R | 6 +- tests/testthat/test-get_user-GitHub.R | 4 +- tests/testthat/test-get_user-GitLab.R | 4 +- 15 files changed, 337 insertions(+), 333 deletions(-) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 966a567c..aa20a36f 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -125,6 +125,45 @@ EngineGraphQLGitHub <- R6::R6Class( return(repos_list_with_commits) }, + # Parses repositories' list with commits into table of commits. + prepare_commits_table = function(repos_list_with_commits, + org) { + commits_table <- purrr::imap(repos_list_with_commits, function(repo, repo_name) { + commits_row <- purrr::map_dfr(repo, function(commit) { + commit_author <- commit$node$author + commit$node$author <- commit_author$name + commit$node$author_login <- if (!is.null(commit_author$user$login)) { + commit_author$user$login + } else { + NA + } + commit$node$author_name <- if (!is.null(commit_author$user$name)) { + commit_author$user$name + } else { + NA + } + commit$node$committed_date <- gts_to_posixt(commit$node$committed_date) + commit$node + }) + commits_row$repository <- repo_name + commits_row + }) %>% + purrr::discard(~ length(.) == 1) %>% + purrr::list_rbind() + if (nrow(commits_table) > 0) { + commits_table <- commits_table %>% + dplyr::mutate( + organization = org, + api_url = self$gql_api_url + ) %>% + dplyr::relocate( + any_of(c("author_login", "author_name")), + .after = author + ) + } + return(commits_table) + }, + # Pull all given files from all repositories of an organization. get_files_from_org = function(org, type, @@ -157,6 +196,30 @@ EngineGraphQLGitHub <- R6::R6Class( return(org_files_list) }, + # Prepare files table. + prepare_files_table = function(files_response, org, file_path) { + if (!is.null(files_response)) { + files_table <- purrr::map(files_response, function(repository) { + purrr::imap(repository, function(file_data, file_name) { + data.frame( + "repo_name" = file_data$repo_name, + "repo_id" = file_data$repo_id, + "organization" = org, + "file_path" = file_name, + "file_content" = file_data$file$text %||% NA, + "file_size" = file_data$file$byteSize, + "repo_url" = file_data$repo_url + ) + }) %>% + purrr::list_rbind() + }) %>% + purrr::list_rbind() + } else { + files_table <- NULL + } + return(files_table) + }, + # Pull all files from all repositories of an organization. get_files_structure_from_org = function(org, type, @@ -186,6 +249,29 @@ EngineGraphQLGitHub <- R6::R6Class( return(files_structure) }, + # Prepare user table. + prepare_user_table = function(user_response) { + if (!is.null(user_response$data$user)) { + user_data <- user_response$data$user + user_data[["name"]] <- user_data$name %||% "" + user_data[["starred_repos"]] <- user_data$starred_repos$totalCount + user_data[["commits"]] <- user_data$contributions$totalCommitContributions + user_data[["issues"]] <- user_data$contributions$totalIssueContributions + user_data[["pull_requests"]] <- user_data$contributions$totalPullRequestContributions + user_data[["reviews"]] <- user_data$contributions$totalPullRequestReviewContributions + user_data[["contributions"]] <- NULL + user_data[["email"]] <- user_data$email %||% "" + user_data[["location"]] <- user_data$location %||% "" + user_data[["web_url"]] <- user_data$web_url %||% "" + user_table <- tibble::as_tibble(user_data) %>% + dplyr::relocate(c(commits, issues, pull_requests, reviews), + .after = starred_repos) + } else { + user_table <- NULL + } + return(user_table) + }, + # Pull release logs from organization get_release_logs_from_org = function(repos_names, org) { release_responses <- purrr::map(repos_names, function(repository) { @@ -201,6 +287,47 @@ EngineGraphQLGitHub <- R6::R6Class( }) %>% purrr::discard(~ length(.$data$repository$releases$nodes) == 0) return(release_responses) + }, + + # Prepare releases table. + prepare_releases_table = function(releases_response, org, date_from, date_until) { + if (!is.null(releases_response)) { + releases_table <- + purrr::map(releases_response, function(release) { + release_table <- purrr::map(release$data$repository$releases$nodes, function(node) { + data.frame( + release_name = node$name, + release_tag = node$tagName, + published_at = gts_to_posixt(node$publishedAt), + release_url = node$url, + release_log = node$description + ) + }) %>% + purrr::list_rbind() %>% + dplyr::mutate( + repo_name = release$data$repository$name, + repo_url = release$data$repository$url + ) %>% + dplyr::relocate( + repo_name, repo_url, + .before = release_name + ) + return(release_table) + }) %>% + purrr::list_rbind() %>% + dplyr::filter( + published_at <= as.POSIXct(date_until) + ) + if (!is.null(date_from)) { + releases_table <- releases_table %>% + dplyr::filter( + published_at >= as.POSIXct(date_from) + ) + } + } else { + releases_table <- NULL + } + return(releases_table) } ), private = list( diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 9b826d3a..3f75ef19 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -270,6 +270,49 @@ EngineGraphQLGitLab <- R6::R6Class( }, .progress = progress) return(org_files_list) }, + + # Prepare files table. + prepare_files_table = function(files_response, org, file_path) { + if (!is.null(files_response)) { + if (private$response_prepared_by_iteration(files_response)) { + files_table <- purrr::map(files_response, function(response_data) { + purrr::map(response_data$data$project$repository$blobs$nodes, function(file) { + data.frame( + "repo_name" = response_data$data$project$name, + "repo_id" = response_data$data$project$id, + "organization" = org, + "file_path" = file$name, + "file_content" = file$rawBlob, + "file_size" = as.integer(file$size), + "repo_url" = response_data$data$project$webUrl + ) + }) %>% + purrr::list_rbind() + }) %>% + purrr::list_rbind() + } else { + files_table <- purrr::map(files_response, function(project) { + purrr::map(project$repository$blobs$nodes, function(file) { + data.frame( + "repo_name" = project$name, + "repo_id" = project$id, + "organization" = org, + "file_path" = file$name, + "file_content" = file$rawBlob, + "file_size" = as.integer(file$size), + "repo_url" = project$webUrl + ) + }) %>% + purrr::list_rbind() + }) %>% + purrr::list_rbind() + } + } else { + files_table <- NULL + } + return(files_table) + }, + get_files_structure_from_org = function(org, type, repos, @@ -296,6 +339,28 @@ EngineGraphQLGitLab <- R6::R6Class( return(files_structure) }, + # Prepare user table. + prepare_user_table = function(user_response) { + if (!is.null(user_response$data$user)) { + user_data <- user_response$data$user + user_data$name <- user_data$name %||% "" + user_data$starred_repos <- user_data$starred_repos$count + user_data$pull_requests <- user_data$pull_requests$count + user_data$reviews <- user_data$reviews$count + user_data$email <- user_data$email %||% "" + user_data$location <- user_data$location %||% "" + user_data$web_url <- user_data$web_url %||% "" + user_table <- tibble::as_tibble(user_data) %>% + dplyr::mutate(commits = NA, + issues = NA) %>% + dplyr::relocate(c(commits, issues), + .after = starred_repos) + } else { + user_table <- NULL + } + return(user_table) + }, + # Pull all releases from all repositories of an organization. get_release_logs_from_org = function(repos_names, org) { release_responses <- purrr::map(repos_names, function(repository) { @@ -310,6 +375,47 @@ EngineGraphQLGitLab <- R6::R6Class( }) %>% purrr::discard(~ length(.$data$project$releases$nodes) == 0) return(release_responses) + }, + + # Prepare releases table. + prepare_releases_table = function(releases_response, org, date_from, date_until) { + if (length(releases_response) > 0) { + releases_table <- + purrr::map(releases_response, function(release) { + release_table <- purrr::map(release$data$project$releases$nodes, function(node) { + data.frame( + release_name = node$name, + release_tag = node$tagName, + published_at = gts_to_posixt(node$releasedAt), + release_url = node$links$selfUrl, + release_log = node$description + ) + }) %>% + purrr::list_rbind() %>% + dplyr::mutate( + repo_name = release$data$project$name, + repo_url = release$data$project$webUrl + ) %>% + dplyr::relocate( + repo_name, repo_url, + .before = release_name + ) + return(release_table) + }) %>% + purrr::list_rbind() %>% + dplyr::filter( + published_at <= as.POSIXct(date_until) + ) + if (!is.null(date_from)) { + releases_table <- releases_table %>% + dplyr::filter( + published_at >= as.POSIXct(date_from) + ) + } + } else { + releases_table <- NULL + } + return(releases_table) } ), private = list( @@ -455,6 +561,10 @@ EngineGraphQLGitLab <- R6::R6Class( "files" = files ) return(result) + }, + + response_prepared_by_iteration = function(files_response) { + !all(purrr::map_lgl(files_response, ~ all(c("name", "id", "webUrl", "repository") %in% names(.)))) } ) ) diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 70683fb2..6d023a0f 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -34,6 +34,29 @@ EngineRestGitLab <- R6::R6Class( return(files_list) }, + # Prepare files table from REST API. + prepare_files_table = function(files_list) { + files_table <- NULL + if (!is.null(files_list)) { + files_table <- purrr::map(files_list, function(file_data) { + org_repo <- stringr::str_split_1(file_data$repo_fullname, "/") + org <- paste0(org_repo[1:(length(org_repo) - 1)], collapse = "/") + data.frame( + "repo_name" = file_data$repo_name, + "repo_id" = as.character(file_data$repo_id), + "organization" = org, + "file_path" = file_data$file_path, + "file_content" = file_data$content, + "file_size" = file_data$size, + "repo_url" = file_data$repo_url + ) + }) %>% + purrr::list_rbind() %>% + unique() + } + return(files_table) + }, + # Wrapper for iteration over GitLab search API response # @details For the time being there is no possibility to search GitLab with # filtering by language. For more information look here: @@ -87,6 +110,45 @@ EngineRestGitLab <- R6::R6Class( return(repos_list) }, + # Get only important info on commits. + tailor_commits_info = function(repos_list_with_commits, + org) { + repos_list_with_commits_cut <- purrr::map(repos_list_with_commits, function(repo) { + purrr::map(repo, function(commit) { + list( + "id" = commit$id, + "committed_date" = gts_to_posixt(commit$committed_date), + "author" = commit$author_name, + "additions" = commit$stats$additions, + "deletions" = commit$stats$deletions, + "repository" = gsub( + pattern = paste0("/-/commit/", commit$id), + replacement = "", + x = gsub(paste0("(.*)", org, "/"), "", commit$web_url) + ), + "organization" = org + ) + }) + }) + return(repos_list_with_commits_cut) + }, + + # A helper to turn list of data.frames into one data.frame + prepare_commits_table = function(commits_list) { + commits_dt <- purrr::map(commits_list, function(commit) { + purrr::map(commit, ~ data.frame(.)) %>% + purrr::list_rbind() + }) %>% + purrr::list_rbind() + if (length(commits_dt) > 0) { + commits_dt <- dplyr::mutate( + commits_dt, + api_url = self$rest_api_url + ) + } + return(commits_dt) + }, + # Pull all repositories URLs from organization get_repos_urls = function(type, org) { repos_urls <- self$response( diff --git a/R/GitHost.R b/R/GitHost.R index f568b8bf..53d84766 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -144,7 +144,7 @@ GitHost <- R6::R6Class( graphql_engine <- private$engines$graphql users_table <- purrr::map(users, function(user) { graphql_engine$get_user(user) %>% - private$prepare_user_table() + graphql_engine$prepare_user_table() }) %>% purrr::list_rbind() return(users_table) @@ -218,13 +218,13 @@ GitHost <- R6::R6Class( repos_names <- private$set_repositories( org = org ) - gql_engine <- private$engines$graphql + graphql_engine <- private$engines$graphql if (length(repos_names) > 0) { - release_logs_table_org <- gql_engine$get_release_logs_from_org( + release_logs_table_org <- graphql_engine$get_release_logs_from_org( org = org, repos_names = repos_names ) %>% - private$prepare_releases_table(org, since, until) + graphql_engine$prepare_releases_table(org, since, until) } else { releases_logs_table_org <- NULL } @@ -887,7 +887,7 @@ GitHost <- R6::R6Class( verbose = verbose, progress = progress ) %>% - private$prepare_files_table( + graphql_engine$prepare_files_table( org = org, file_path = file_path ) diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 7f0a6c3c..21b4c8e8 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -185,7 +185,7 @@ GitHostGitHub <- R6::R6Class( until = until, progress = progress ) %>% - private$prepare_commits_table(org) + graphql_engine$prepare_commits_table(org) return(commits_table_org) }, .progress = if (private$scan_all && progress) { "[GitHost:GitHub] Pulling commits..." @@ -209,92 +209,6 @@ GitHostGitHub <- R6::R6Class( return(repos_names) }, - # Parses repositories' list with commits into table of commits. - prepare_commits_table = function(repos_list_with_commits, - org) { - commits_table <- purrr::imap(repos_list_with_commits, function(repo, repo_name) { - commits_row <- purrr::map_dfr(repo, function(commit) { - commit_author <- commit$node$author - commit$node$author <- commit_author$name - commit$node$author_login <- if (!is.null(commit_author$user$login)) { - commit_author$user$login - } else { - NA - } - commit$node$author_name <- if (!is.null(commit_author$user$name)) { - commit_author$user$name - } else { - NA - } - commit$node$committed_date <- gts_to_posixt(commit$node$committed_date) - commit$node - }) - commits_row$repository <- repo_name - commits_row - }) %>% - purrr::discard(~ length(.) == 1) %>% - purrr::list_rbind() - if (nrow(commits_table) > 0) { - commits_table <- commits_table %>% - dplyr::mutate( - organization = org, - api_url = private$api_url - ) %>% - dplyr::relocate( - any_of(c("author_login", "author_name")), - .after = author - ) - } - return(commits_table) - }, - - # Prepare user table. - prepare_user_table = function(user_response) { - if (!is.null(user_response$data$user)) { - user_data <- user_response$data$user - user_data[["name"]] <- user_data$name %||% "" - user_data[["starred_repos"]] <- user_data$starred_repos$totalCount - user_data[["commits"]] <- user_data$contributions$totalCommitContributions - user_data[["issues"]] <- user_data$contributions$totalIssueContributions - user_data[["pull_requests"]] <- user_data$contributions$totalPullRequestContributions - user_data[["reviews"]] <- user_data$contributions$totalPullRequestReviewContributions - user_data[["contributions"]] <- NULL - user_data[["email"]] <- user_data$email %||% "" - user_data[["location"]] <- user_data$location %||% "" - user_data[["web_url"]] <- user_data$web_url %||% "" - user_table <- tibble::as_tibble(user_data) %>% - dplyr::relocate(c(commits, issues, pull_requests, reviews), - .after = starred_repos) - } else { - user_table <- NULL - } - return(user_table) - }, - - # Prepare files table. - prepare_files_table = function(files_response, org, file_path) { - if (!is.null(files_response)) { - files_table <- purrr::map(files_response, function(repository) { - purrr::imap(repository, function(file_data, file_name) { - data.frame( - "repo_name" = file_data$repo_name, - "repo_id" = file_data$repo_id, - "organization" = org, - "file_path" = file_name, - "file_content" = file_data$file$text %||% NA, - "file_size" = file_data$file$byteSize, - "repo_url" = file_data$repo_url - ) - }) %>% - purrr::list_rbind() - }) %>% - purrr::list_rbind() - } else { - files_table <- NULL - } - return(files_table) - }, - # Prepare files table from REST API. prepare_files_table_from_rest = function(files_list) { files_table <- NULL @@ -327,47 +241,6 @@ GitHostGitHub <- R6::R6Class( # Get repository url set_repo_url = function(repo_fullname) { paste0(private$endpoints$repositories, "/", repo_fullname) - }, - - # Prepare releases table. - prepare_releases_table = function(releases_response, org, date_from, date_until) { - if (!is.null(releases_response)) { - releases_table <- - purrr::map(releases_response, function(release) { - release_table <- purrr::map(release$data$repository$releases$nodes, function(node) { - data.frame( - release_name = node$name, - release_tag = node$tagName, - published_at = gts_to_posixt(node$publishedAt), - release_url = node$url, - release_log = node$description - ) - }) %>% - purrr::list_rbind() %>% - dplyr::mutate( - repo_name = release$data$repository$name, - repo_url = release$data$repository$url - ) %>% - dplyr::relocate( - repo_name, repo_url, - .before = release_name - ) - return(release_table) - }) %>% - purrr::list_rbind() %>% - dplyr::filter( - published_at <= as.POSIXct(date_until) - ) - if (!is.null(date_from)) { - releases_table <- releases_table %>% - dplyr::filter( - published_at >= as.POSIXct(date_from) - ) - } - } else { - releases_table <- NULL - } - return(releases_table) } ) ) diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index 281fe017..428b6e0f 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -266,8 +266,8 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", until = until, progress = progress ) %>% - private$tailor_commits_info(org = org) %>% - private$prepare_commits_table() %>% + rest_engine$tailor_commits_info(org = org) %>% + rest_engine$prepare_commits_table() %>% rest_engine$get_commits_authors_handles_and_names( verbose = verbose, progress = progress @@ -298,46 +298,6 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", return(repos_names) }, - # Get only important info on commits. - tailor_commits_info = function(repos_list_with_commits, - org) { - repos_list_with_commits_cut <- purrr::map(repos_list_with_commits, function(repo) { - purrr::map(repo, function(commit) { - list( - "id" = commit$id, - "committed_date" = gts_to_posixt(commit$committed_date), - "author" = commit$author_name, - "additions" = commit$stats$additions, - "deletions" = commit$stats$deletions, - "repository" = gsub( - pattern = paste0("/-/commit/", commit$id), - replacement = "", - x = gsub(paste0("(.*)", org, "/"), "", commit$web_url) - ), - "organization" = org - ) - }) - }) - return(repos_list_with_commits_cut) - }, - - # A helper to turn list of data.frames into one data.frame - prepare_commits_table = function(commits_list) { - commits_dt <- purrr::map(commits_list, function(commit) { - purrr::map(commit, ~ data.frame(.)) %>% - purrr::list_rbind() - }) %>% - purrr::list_rbind() - - if (length(commits_dt) > 0) { - commits_dt <- dplyr::mutate( - commits_dt, - api_url = private$api_url - ) - } - return(commits_dt) - }, - are_non_text_files = function(file_path, host_files_structure) { if (!is.null(file_path)) { any(grepl(non_text_files_pattern, file_path)) @@ -395,143 +355,11 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", verbose = FALSE, progress = progress ) %>% - private$prepare_files_table_from_rest() + rest_engine$prepare_files_table() }, .progress = progress) %>% purrr::list_rbind() %>% private$add_repo_api_url() return(files_table) - }, - - # Prepare user table. - prepare_user_table = function(user_response) { - if (!is.null(user_response$data$user)) { - user_data <- user_response$data$user - user_data$name <- user_data$name %||% "" - user_data$starred_repos <- user_data$starred_repos$count - user_data$pull_requests <- user_data$pull_requests$count - user_data$reviews <- user_data$reviews$count - user_data$email <- user_data$email %||% "" - user_data$location <- user_data$location %||% "" - user_data$web_url <- user_data$web_url %||% "" - user_table <- tibble::as_tibble(user_data) %>% - dplyr::mutate(commits = NA, - issues = NA) %>% - dplyr::relocate(c(commits, issues), - .after = starred_repos) - } else { - user_table <- NULL - } - return(user_table) - }, - - # Prepare files table. - prepare_files_table = function(files_response, org, file_path) { - if (!is.null(files_response)) { - if (private$response_prepared_by_iteration(files_response)) { - files_table <- purrr::map(files_response, function(response_data) { - purrr::map(response_data$data$project$repository$blobs$nodes, function(file) { - data.frame( - "repo_name" = response_data$data$project$name, - "repo_id" = response_data$data$project$id, - "organization" = org, - "file_path" = file$name, - "file_content" = file$rawBlob, - "file_size" = as.integer(file$size), - "repo_url" = response_data$data$project$webUrl - ) - }) %>% - purrr::list_rbind() - }) %>% - purrr::list_rbind() - } else { - files_table <- purrr::map(files_response, function(project) { - purrr::map(project$repository$blobs$nodes, function(file) { - data.frame( - "repo_name" = project$name, - "repo_id" = project$id, - "organization" = org, - "file_path" = file$name, - "file_content" = file$rawBlob, - "file_size" = as.integer(file$size), - "repo_url" = project$webUrl - ) - }) %>% - purrr::list_rbind() - }) %>% - purrr::list_rbind() - } - } else { - files_table <- NULL - } - return(files_table) - }, - - response_prepared_by_iteration = function(files_response) { - !all(purrr::map_lgl(files_response, ~ all(c("name", "id", "webUrl", "repository") %in% names(.)))) - }, - - # Prepare files table from REST API. - prepare_files_table_from_rest = function(files_list) { - files_table <- NULL - if (!is.null(files_list)) { - files_table <- purrr::map(files_list, function(file_data) { - org_repo <- stringr::str_split_1(file_data$repo_fullname, "/") - org <- paste0(org_repo[1:(length(org_repo) - 1)], collapse = "/") - data.frame( - "repo_name" = file_data$repo_name, - "repo_id" = as.character(file_data$repo_id), - "organization" = org, - "file_path" = file_data$file_path, - "file_content" = file_data$content, - "file_size" = file_data$size, - "repo_url" = file_data$repo_url - ) - }) %>% - purrr::list_rbind() %>% - unique() - } - return(files_table) - }, - - # Prepare releases table. - prepare_releases_table = function(releases_response, org, date_from, date_until) { - if (length(releases_response) > 0) { - releases_table <- - purrr::map(releases_response, function(release) { - release_table <- purrr::map(release$data$project$releases$nodes, function(node) { - data.frame( - release_name = node$name, - release_tag = node$tagName, - published_at = gts_to_posixt(node$releasedAt), - release_url = node$links$selfUrl, - release_log = node$description - ) - }) %>% - purrr::list_rbind() %>% - dplyr::mutate( - repo_name = release$data$project$name, - repo_url = release$data$project$webUrl - ) %>% - dplyr::relocate( - repo_name, repo_url, - .before = release_name - ) - return(release_table) - }) %>% - purrr::list_rbind() %>% - dplyr::filter( - published_at <= as.POSIXct(date_until) - ) - if (!is.null(date_from)) { - releases_table <- releases_table %>% - dplyr::filter( - published_at >= as.POSIXct(date_from) - ) - } - } else { - releases_table <- NULL - } - return(releases_table) } ) ) diff --git a/R/GitStats.R b/R/GitStats.R index 3e75eb60..27ff5759 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -833,7 +833,8 @@ GitStats <- R6::R6Class( progress = progress ) }) %>% - purrr::list_rbind() + purrr::list_rbind() %>% + dplyr::as_tibble() return(commits_table) }, @@ -843,7 +844,8 @@ GitStats <- R6::R6Class( host$get_users(logins) }) %>% unique() %>% - purrr::list_rbind() + purrr::list_rbind() %>% + dplyr::as_tibble() }, # Pull content of a text file in a table form @@ -879,7 +881,8 @@ GitStats <- R6::R6Class( ) } }) %>% - purrr::list_rbind() + purrr::list_rbind() %>% + dplyr::as_tibble() }, get_host_files_structure = function(host, verbose) { @@ -943,7 +946,8 @@ GitStats <- R6::R6Class( progress = progress ) }) %>% - purrr::list_rbind() + purrr::list_rbind() %>% + dplyr::as_tibble() }, # Pull information on package usage in a table form diff --git a/tests/testthat/test-get_commits-GitHub.R b/tests/testthat/test-get_commits-GitHub.R index 88f7851f..c8b4e6e3 100644 --- a/tests/testthat/test-get_commits-GitHub.R +++ b/tests/testthat/test-get_commits-GitHub.R @@ -68,7 +68,7 @@ test_that("`get_commits_from_repos()` pulls commits from repos", { }) test_that("`prepare_commits_table()` prepares commits table", { - gh_commits_table <- github_testhost_priv$prepare_commits_table( + gh_commits_table <- test_graphql_github$prepare_commits_table( repos_list_with_commits = test_mocker$use("commits_from_repos"), org = "r-world-devs" ) @@ -81,7 +81,7 @@ test_that("`prepare_commits_table()` prepares commits table", { test_that("get_commits_from_orgs for GitHub works", { mockery::stub( github_testhost_repos_priv$get_commits_from_orgs, - "private$prepare_commits_table", + "graphql_engine$prepare_commits_table", test_mocker$use("gh_commits_table") ) suppressMessages( diff --git a/tests/testthat/test-get_commits-GitLab.R b/tests/testthat/test-get_commits-GitLab.R index b0113ab7..d75360e0 100644 --- a/tests/testthat/test-get_commits-GitLab.R +++ b/tests/testthat/test-get_commits-GitLab.R @@ -18,7 +18,7 @@ test_that("`get_commits_from_repos()` pulls commits from repo", { test_that("`tailor_commits_info()` retrieves only necessary info", { gl_commits_list <- test_mocker$use("gl_commits_org") - gl_commits_list_cut <- gitlab_testhost_priv$tailor_commits_info( + gl_commits_list_cut <- test_rest_gitlab$tailor_commits_info( gl_commits_list, org = "mbtests" ) @@ -29,7 +29,7 @@ test_that("`tailor_commits_info()` retrieves only necessary info", { }) test_that("`prepare_commits_table()` prepares table of commits properly", { - gl_commits_table <- gitlab_testhost_priv$prepare_commits_table( + gl_commits_table <- test_rest_gitlab$prepare_commits_table( commits_list = test_mocker$use("gl_commits_list_cut") ) expect_commits_table( diff --git a/tests/testthat/test-get_files_content-GitHub.R b/tests/testthat/test-get_files_content-GitHub.R index 4d3a9423..ec9183c8 100644 --- a/tests/testthat/test-get_files_content-GitHub.R +++ b/tests/testthat/test-get_files_content-GitHub.R @@ -108,7 +108,7 @@ test_that("GitHub GraphQL Engine pulls files from organization", { }) test_that("GitHubHost prepares table from files response", { - gh_files_table <- github_testhost_priv$prepare_files_table( + gh_files_table <- test_graphql_github$prepare_files_table( files_response = test_mocker$use("github_files_response"), org = "r-world-devs", file_path = "LICENSE" @@ -123,7 +123,7 @@ test_that("GitHubHost prepares table from files with no content", { test_repo$test_files$file$text <- NULL return(test_repo) }) - gh_empty_files_table <- github_testhost_priv$prepare_files_table( + gh_empty_files_table <- test_graphql_github$prepare_files_table( files_response = empty_files_response, org = "test_org", file_path = "test_files" @@ -136,7 +136,7 @@ test_that("GitHubHost prepares table from files with no content", { test_that("get_files_content_from_orgs for GitHub works", { mockery::stub( github_testhost_priv$get_files_content_from_orgs, - "private$prepare_files_table", + "graphql_engine$prepare_files_table", test_mocker$use("gh_files_table") ) gh_files_table <- github_testhost_priv$get_files_content_from_orgs( diff --git a/tests/testthat/test-get_files_content-GitLab.R b/tests/testthat/test-get_files_content-GitLab.R index c034666b..45e40f38 100644 --- a/tests/testthat/test-get_files_content-GitLab.R +++ b/tests/testthat/test-get_files_content-GitLab.R @@ -132,19 +132,19 @@ test_that("Gitlab GraphQL switches to pulling files per repositories when query test_that("checker properly identifies gitlab files responses", { expect_false( - gitlab_testhost_priv$response_prepared_by_iteration( + test_graphql_gitlab_priv$response_prepared_by_iteration( files_response = test_mocker$use("gitlab_files_response") ) ) expect_true( - gitlab_testhost_priv$response_prepared_by_iteration( + test_graphql_gitlab_priv$response_prepared_by_iteration( files_response = test_mocker$use("gitlab_files_response_by_repos") ) ) }) test_that("GitLab prepares table from files response", { - gl_files_table <- gitlab_testhost_priv$prepare_files_table( + gl_files_table <- test_graphql_gitlab$prepare_files_table( files_response = test_mocker$use("gitlab_files_response"), org = "mbtests", file_path = "meta_data.yaml" @@ -154,7 +154,7 @@ test_that("GitLab prepares table from files response", { }) test_that("GitLab prepares table from files response prepared in alternative way", { - gl_files_table <- gitlab_testhost_priv$prepare_files_table( + gl_files_table <- test_graphql_gitlab$prepare_files_table( files_response = test_mocker$use("gitlab_files_response_by_repos"), org = "mbtests", file_path = "meta_data.yaml" @@ -165,7 +165,7 @@ test_that("GitLab prepares table from files response prepared in alternative way test_that("get_files_content_from_orgs for GitLab works", { mockery::stub( gitlab_testhost_priv$get_files_content_from_orgs, - "private$prepare_files_table", + "graphql_engine$prepare_files_table", test_mocker$use("gl_files_table") ) suppressMessages( diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index 1b2bd73d..3c93c179 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -21,7 +21,7 @@ test_that("`get_releases_from_org()` pulls releases from the repositories", { }) test_that("`prepare_releases_table()` prepares releases table", { - releases_table <- github_testhost_priv$prepare_releases_table( + releases_table <- test_graphql_github$prepare_releases_table( releases_response = test_mocker$use("releases_from_repos"), org = "r-world-devs", date_from = "2023-05-01", @@ -48,7 +48,7 @@ test_that("`set_repositories` works", { test_that("`get_release_logs()` pulls release logs in the table format", { mockery::stub( github_testhost$get_release_logs, - "private$prepare_releases_table", + "graphql_engine$prepare_releases_table", test_mocker$use("releases_table") ) mockery::stub( diff --git a/tests/testthat/test-get_release-GitLab.R b/tests/testthat/test-get_release-GitLab.R index 15b18ca1..fe79a434 100644 --- a/tests/testthat/test-get_release-GitLab.R +++ b/tests/testthat/test-get_release-GitLab.R @@ -22,9 +22,9 @@ test_that("`get_releases_from_org()` pulls releases from the repositories", { test_that("`prepare_releases_table()` prepares releases table", { - releases_table <- gitlab_testhost_priv$prepare_releases_table( + releases_table <- test_graphql_gitlab$prepare_releases_table( releases_response = test_mocker$use("releases_from_repos"), - org = "r-world-devs", + org = "test_org", date_from = "2023-08-01", date_until = "2024-06-30" ) @@ -37,7 +37,7 @@ test_that("`prepare_releases_table()` prepares releases table", { test_that("`get_release_logs()` pulls release logs in the table format", { mockery::stub( gitlab_testhost$get_release_logs, - "private$prepare_releases_table", + "graphql_engine$prepare_releases_table", test_mocker$use("releases_table") ) releases_table <- gitlab_testhost$get_release_logs( diff --git a/tests/testthat/test-get_user-GitHub.R b/tests/testthat/test-get_user-GitHub.R index 250d6c3c..bba98a84 100644 --- a/tests/testthat/test-get_user-GitHub.R +++ b/tests/testthat/test-get_user-GitHub.R @@ -21,7 +21,7 @@ test_that("get_user pulls GitHub user response", { }) test_that("GitHub prepares user table", { - gh_user_table <- github_testhost_priv$prepare_user_table( + gh_user_table <- test_graphql_github$prepare_user_table( user_response = test_mocker$use("gh_user_response") ) expect_users_table( @@ -34,7 +34,7 @@ test_that("GitHub prepares user table", { test_that("GitHost gets users tables", { mockery::stub( github_testhost$get_users, - "private$prepare_user_table", + "graphql_engine$prepare_user_table", test_mocker$use("gh_user_table") ) github_users <- github_testhost$get_users( diff --git a/tests/testthat/test-get_user-GitLab.R b/tests/testthat/test-get_user-GitLab.R index 4b7bed2f..9ecea4f4 100644 --- a/tests/testthat/test-get_user-GitLab.R +++ b/tests/testthat/test-get_user-GitLab.R @@ -21,7 +21,7 @@ test_that("get_user pulls GitLab user response", { }) test_that("GitLab prepares user table", { - gl_user_table <- gitlab_testhost_priv$prepare_user_table( + gl_user_table <- test_graphql_gitlab$prepare_user_table( user_response = test_mocker$use("gl_user_response") ) expect_users_table( @@ -34,7 +34,7 @@ test_that("GitLab prepares user table", { test_that("get_users build users table for GitLab", { mockery::stub( gitlab_testhost$get_users, - "private$prepare_user_table", + "graphql_engine$prepare_user_table", test_mocker$use("gl_user_table") ) gitlab_users <- gitlab_testhost$get_users( From 9defa4c413e4fc5e32d4d18c04a2c70e1b5cda0c Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 15 Oct 2024 12:21:41 +0000 Subject: [PATCH 49/94] Bump version. --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 971b2a0f..7782078c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.0.9002 +Version: 2.1.0.9003 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), From 4451c104e717261bdcd777f1d9c0c930074132de Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 15 Oct 2024 14:11:38 +0000 Subject: [PATCH 50/94] Adjust properly api url setter when public host name is passed. --- DESCRIPTION | 2 +- NEWS.md | 1 + R/GitHostGitHub.R | 4 +++- tests/testthat/test-GitHost-helpers.R | 28 +++++++++++++++++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7782078c..d2bd5b57 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.0.9003 +Version: 2.1.0.9004 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), diff --git a/NEWS.md b/NEWS.md index 9c432831..2b101654 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,7 @@ - Added possibility to get repositories for individual users with `get_repos()` ([#492](https://github.com/r-world-devs/GitStats/issues/492)). Earlier this was only possible for GitHub organizations and GitLab groups. - Fixed getting large search responses for GitHub ([#491](https://github.com/r-world-devs/GitStats/issues/491)). - Fixed checking token scopes ([#501](https://github.com/r-world-devs/GitStats/issues/501)). If token scopes are insufficient error is returned and `GitHost` is not passed to `GitStats`. This also applies to situation when `GitStats` looks for default tokens (not defined by user). Earlier, if tests for token failed, an empty token was passed and `GitStats` was created, which was misleading for the user. +- User can now optionally pass public GitHub host name (`github.com` or `https://github.com`) to `set_github_host()` ([#475](https://github.com/r-world-devs/GitStats/issues/475)). # GitStats 2.1.0 diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 21b4c8e8..24cdc67f 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -54,7 +54,9 @@ GitHostGitHub <- R6::R6Class( # Set API URL set_api_url = function(host) { - if (is.null(host)) { + if (is.null(host) || + host == "https://github.com" || + host == "github.com") { private$api_url <- "https://api.github.com" } else { private$set_custom_api_url(host) diff --git a/tests/testthat/test-GitHost-helpers.R b/tests/testthat/test-GitHost-helpers.R index ade90edd..d42e58fe 100644 --- a/tests/testthat/test-GitHost-helpers.R +++ b/tests/testthat/test-GitHost-helpers.R @@ -21,3 +21,31 @@ test_that("set_owner_types sets attributes to owners list", { expect_equal(attr(owner[[1]], "type"), "organization") expect_equal(owner[[1]], "test_org", ignore_attr = TRUE) }) + +test_that("set_api_url works for public hosts", { + expect_equal({ + github_testhost_priv$set_api_url( + host = "github.com" + ) + }, "https://api.github.com") + expect_equal({ + github_testhost_priv$set_api_url( + host = "https://github.com" + ) + }, "https://api.github.com") + expect_equal({ + github_testhost_priv$set_api_url( + host = "https://github.company.com" + ) + }, "https://github.company.com/api/v3") + expect_equal({ + gitlab_testhost_priv$set_api_url( + host = "https://gitlab.com" + ) + }, "https://gitlab.com/api/v4") + expect_equal({ + gitlab_testhost_priv$set_api_url( + host = "gitlab.com" + ) + }, "https://gitlab.com/api/v4") +}) From 5f28318951b2c880b3938df2de37eabf19608386 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 15 Oct 2024 14:12:13 +0000 Subject: [PATCH 51/94] Skip tests on CRAN as they use api token. --- tests/testthat/test-set_host.R | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/testthat/test-set_host.R b/tests/testthat/test-set_host.R index 0aaba037..8329187a 100644 --- a/tests/testthat/test-set_host.R +++ b/tests/testthat/test-set_host.R @@ -1,6 +1,7 @@ test_gitstats <- create_gitstats() test_that("Set connection returns appropriate messages", { + skip_on_cran() expect_snapshot( set_github_host( gitstats_obj = test_gitstats, @@ -17,6 +18,7 @@ test_that("Set connection returns appropriate messages", { }) test_that("When empty token for GitHub, GitStats pulls default token", { + skip_on_cran() expect_snapshot( test_gitstats <- create_gitstats() %>% set_github_host( @@ -26,6 +28,7 @@ test_that("When empty token for GitHub, GitStats pulls default token", { }) test_that("When empty token for GitLab, GitStats pulls default token", { + skip_on_cran() expect_snapshot( withr::with_envvar(new = c("GITLAB_PAT" = Sys.getenv("GITLAB_PAT_PUBLIC")), { test_gitstats <- create_gitstats() %>% @@ -37,6 +40,7 @@ test_that("When empty token for GitLab, GitStats pulls default token", { }) test_that("Set GitHub host with particular repos vector instead of orgs", { + skip_on_cran() test_gitstats <- create_gitstats() expect_snapshot( test_gitstats %>% @@ -52,6 +56,7 @@ test_that("Set GitHub host with particular repos vector instead of orgs", { }) test_that("Set GitLab host with particular repos vector instead of orgs", { + skip_on_cran() test_gitstats <- create_gitstats() expect_snapshot( test_gitstats %>% @@ -67,6 +72,7 @@ test_that("Set GitLab host with particular repos vector instead of orgs", { }) test_that("Set host prints error when repos and orgs are defined and host is not passed to GitStats", { + skip_on_cran() test_gitstats <- create_gitstats() expect_snapshot_error( test_gitstats %>% @@ -83,6 +89,7 @@ test_that("Set host prints error when repos and orgs are defined and host is not }) test_that("Error shows if organizations are not specified and host is not passed", { + skip_on_cran() test_gitstats <- create_gitstats() expect_snapshot_error( test_gitstats %>% @@ -97,6 +104,7 @@ test_that("Error shows if organizations are not specified and host is not passed }) test_that("Error shows, when wrong input is passed when setting connection and host is not passed", { + skip_on_cran() test_gitstats <- create_gitstats() expect_snapshot_error( set_gitlab_host( @@ -117,8 +125,8 @@ test_that("Error shows, when wrong input is passed when setting connection and h }) test_that("Error pops out, when two clients of the same url api are passed as input", { + skip_on_cran() test_gitstats <- create_gitstats() - expect_snapshot( error = TRUE, test_gitstats %>% @@ -134,6 +142,7 @@ test_that("Error pops out, when two clients of the same url api are passed as in }) test_that("Error pops out when `org` does not exist", { + skip_on_cran() expect_snapshot({ test_gitstats <- create_gitstats() %>% set_github_host( From 95ab021b5ad0c9ed54a27efb3bae8c44b70688ee Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 17 Oct 2024 07:40:16 +0000 Subject: [PATCH 52/94] Optimize get_R_package_usage: make possible passing vector of package names to function, change name of the main parameter to `packages` and make the output more concise. The consequence of second feature was introduction of new parameter `output` instead of `raw_output` in private methods handling repositories responses. --- DESCRIPTION | 2 +- NEWS.md | 8 + R/EngineRest.R | 29 ++- R/EngineRestGitHub.R | 57 +++-- R/EngineRestGitLab.R | 64 +++-- R/GitHost.R | 197 ++++++++------- R/GitStats.R | 134 ++++++---- R/gitstats_functions.R | 6 +- inst/package_usage_workflow.R | 13 + man/get_R_package_usage.Rd | 4 +- tests/testthat/_snaps/01-get_repos-GitHub.md | 17 +- tests/testthat/_snaps/get_usage_R_package.md | 4 +- tests/testthat/helper-expect-tables.R | 13 +- tests/testthat/test-01-get_repos-GitHub.R | 251 +++++++++++++++++-- tests/testthat/test-01-get_repos-GitStats.R | 42 ++-- tests/testthat/test-get_usage_R_package.R | 63 ++++- vignettes/get_repos_with_code.Rmd | 4 +- 17 files changed, 632 insertions(+), 276 deletions(-) create mode 100644 inst/package_usage_workflow.R diff --git a/DESCRIPTION b/DESCRIPTION index d2bd5b57..99e7d83e 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.0.9004 +Version: 2.1.0.9005 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), diff --git a/NEWS.md b/NEWS.md index 2b101654..911cb248 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,14 @@ # GitStats (development version) +## Features: + +- Optimized `get_R_package_usage()` function: + - you can now pass a vector of packages names (new `packages` parameter replacing old `package_name`) ([#494](https://github.com/r-world-devs/GitStats/issues/494)), + - on the other hand, output of the function has been limited to contain only most necessary data (removing all repository stats), making thus process of obtaining package usage faster ([#474](https://github.com/r-world-devs/GitStats/issues/474)). - Added possibility to get repositories for individual users with `get_repos()` ([#492](https://github.com/r-world-devs/GitStats/issues/492)). Earlier this was only possible for GitHub organizations and GitLab groups. + +## Fixes: + - Fixed getting large search responses for GitHub ([#491](https://github.com/r-world-devs/GitStats/issues/491)). - Fixed checking token scopes ([#501](https://github.com/r-world-devs/GitStats/issues/501)). If token scopes are insufficient error is returned and `GitHost` is not passed to `GitStats`. This also applies to situation when `GitStats` looks for default tokens (not defined by user). Earlier, if tests for token failed, an empty token was passed and `GitStats` was created, which was misleading for the user. - User can now optionally pass public GitHub host name (`github.com` or `https://github.com`) to `set_github_host()` ([#475](https://github.com/r-world-devs/GitStats/issues/475)). diff --git a/R/EngineRest.R b/R/EngineRest.R index 6b8ad1fc..c6bea57d 100644 --- a/R/EngineRest.R +++ b/R/EngineRest.R @@ -39,7 +39,7 @@ EngineRest <- R6::R6Class("EngineRest", }, # Prepare table for repositories content - prepare_repos_table = function(repos_list, verbose = TRUE) { + prepare_repos_table = function(repos_list, output = "table_full", verbose = TRUE) { repos_dt <- purrr::map(repos_list, function(repo) { repo <- purrr::map(repo, function(attr) { attr <- attr %||% "" @@ -51,15 +51,24 @@ EngineRest <- R6::R6Class("EngineRest", cli::cli_alert_info("Preparing repositories table...") } if (length(repos_dt) > 0) { - repos_dt <- dplyr::mutate( - repos_dt, - repo_id = as.character(repo_id), - created_at = as.POSIXct(created_at), - last_activity_at = as.POSIXct(last_activity_at), - forks = as.integer(forks), - issues_open = as.integer(issues_open), - issues_closed = as.integer(issues_closed) - ) + if (output == "table_full") { + repos_dt <- dplyr::mutate( + repos_dt, + repo_id = as.character(repo_id), + created_at = as.POSIXct(created_at), + last_activity_at = as.POSIXct(last_activity_at), + forks = as.integer(forks), + issues_open = as.integer(issues_open), + issues_closed = as.integer(issues_closed) + ) + } + if (output == "table_min") { + repos_dt <- dplyr::mutate( + repos_dt, + repo_id = as.character(repo_id), + created_at = as.POSIXct(created_at) + ) + } } return(repos_dt) } diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index 7ddd1784..5b8a69b1 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -29,7 +29,7 @@ EngineRestGitHub <- R6::R6Class( org = NULL, filename = NULL, in_path = FALSE, - raw_output = FALSE, + output = "table_full", verbose = TRUE, progress = TRUE) { user_query <- if (!is.null(org)) { @@ -53,12 +53,12 @@ EngineRestGitHub <- R6::R6Class( search_endpoint = search_endpoint, total_n = total_n ) - if (!raw_output) { + if (output == "table_full" || output == "table_min") { search_output <- private$map_search_into_repos( search_response = search_result, progress = progress ) - } else { + } else if (output == "raw") { search_output <- search_result } } else { @@ -68,26 +68,39 @@ EngineRestGitHub <- R6::R6Class( }, # Retrieve only important info from repositories response - tailor_repos_response = function(repos_response) { + tailor_repos_response = function(repos_response, output = "table_full") { repos_list <- purrr::map(repos_response, function(repo) { - list( - "repo_id" = repo$id, - "repo_name" = repo$name, - "default_branch" = repo$default_branch, - "stars" = repo$stargazers_count, - "forks" = repo$forks_count, - "created_at" = gts_to_posixt(repo$created_at), - "last_activity_at" = if (!is.null(repo$pushed_at)) { - gts_to_posixt(repo$pushed_at) - } else { - gts_to_posixt(repo$created_at) - }, - "languages" = repo$language, - "issues_open" = repo$issues_open, - "issues_closed" = repo$issues_closed, - "organization" = repo$owner$login, - "repo_url" = repo$html_url - ) + if (output == "table_full") { + repo_data <- list( + "repo_id" = repo$id, + "repo_name" = repo$name, + "default_branch" = repo$default_branch, + "stars" = repo$stargazers_count, + "forks" = repo$forks_count, + "created_at" = gts_to_posixt(repo$created_at), + "last_activity_at" = if (!is.null(repo$pushed_at)) { + gts_to_posixt(repo$pushed_at) + } else { + gts_to_posixt(repo$created_at) + }, + "languages" = repo$language, + "issues_open" = repo$issues_open, + "issues_closed" = repo$issues_closed, + "organization" = repo$owner$login, + "repo_url" = repo$html_url + ) + } + if (output == "table_min") { + repo_data <- list( + "repo_id" = repo$id, + "repo_name" = repo$name, + "default_branch" = repo$default_branch, + "created_at" = gts_to_posixt(repo$created_at), + "organization" = repo$owner$login, + "repo_url" = repo$html_url + ) + } + return(repo_data) }) return(repos_list) }, diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 6d023a0f..a11048ce 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -62,11 +62,11 @@ EngineRestGitLab <- R6::R6Class( # filtering by language. For more information look here: # https://gitlab.com/gitlab-org/gitlab/-/issues/340333 get_repos_by_code = function(code, - org = NULL, + org = NULL, filename = NULL, - in_path = FALSE, - raw_output = FALSE, - verbose = TRUE, + in_path = FALSE, + output = "table_full", + verbose = TRUE, progress = TRUE) { search_response <- private$search_for_code( code = code, @@ -75,37 +75,53 @@ EngineRestGitLab <- R6::R6Class( org = org, verbose = verbose ) - if (raw_output) { + if (output == "raw") { search_output <- search_response - } else { + } else if (output == "table_full" || output == "table_min") { search_output <- search_response %>% private$map_search_into_repos( progress = progress - ) %>% - private$get_repos_languages( - progress = progress ) + if (output == "table_full") { + search_output <- search_output %>% + private$get_repos_languages( + progress = progress + ) + } } return(search_output) }, # Retrieve only important info from repositories response - tailor_repos_response = function(repos_response) { + tailor_repos_response = function(repos_response, output = "table_full") { repos_list <- purrr::map(repos_response, function(project) { - list( - "repo_id" = project$id, - "repo_name" = project$name, - "default_branch" = project$default_branch, - "stars" = project$star_count, - "forks" = project$fork_count, - "created_at" = project$created_at, - "last_activity_at" = project$last_activity_at, - "languages" = paste0(project$languages, collapse = ", "), - "issues_open" = project$issues_open, - "issues_closed" = project$issues_closed, - "organization" = project$namespace$path, - "repo_url" = project$web_url - ) + if (output == "table_full") { + repo_data <- list( + "repo_id" = project$id, + "repo_name" = project$name, + "default_branch" = project$default_branch, + "stars" = project$star_count, + "forks" = project$fork_count, + "created_at" = project$created_at, + "last_activity_at" = project$last_activity_at, + "languages" = paste0(project$languages, collapse = ", "), + "issues_open" = project$issues_open, + "issues_closed" = project$issues_closed, + "organization" = project$namespace$path, + "repo_url" = project$web_url + ) + } + if (output == "table_min") { + repo_data <- list( + "repo_id" = project$id, + "repo_name" = project$name, + "default_branch" = project$default_branch, + "created_at" = project$created_at, + "organization" = project$namespace$path, + "repo_url" = project$web_url + ) + } + return(repo_data) }) return(repos_list) }, diff --git a/R/GitHost.R b/R/GitHost.R index 53d84766..4717d5d2 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -45,6 +45,7 @@ GitHost <- R6::R6Class( with_code = NULL, in_files = NULL, with_file = NULL, + output = "table_full", verbose = TRUE, progress = TRUE) { if (is.null(with_code) && is.null(with_file)) { @@ -57,6 +58,7 @@ GitHost <- R6::R6Class( repos_table <- private$get_repos_with_code( code = with_code, in_files = in_files, + output = output, verbose = verbose, progress = progress ) @@ -64,17 +66,21 @@ GitHost <- R6::R6Class( repos_table <- private$get_repos_with_code( code = with_file, in_path = TRUE, + output = output, verbose = verbose, progress = progress ) } - repos_table <- private$add_repo_api_url(repos_table) - if (add_contributors) { - repos_table <- private$get_repos_contributors( - repos_table = repos_table, - verbose = verbose, - progress = progress - ) + if (output == "table_full" || output == "table_min") { + repos_table <- private$add_repo_api_url(repos_table) %>% + private$add_platform() + if (add_contributors) { + repos_table <- private$get_repos_contributors( + repos_table = repos_table, + verbose = verbose, + progress = progress + ) + } } return(repos_table) }, @@ -88,10 +94,10 @@ GitHost <- R6::R6Class( progress = TRUE) { if (!is.null(with_code)) { repo_urls <- private$get_repos_with_code( - code = with_code, - in_files = in_files, - raw_output = TRUE, - verbose = verbose + code = with_code, + in_files = in_files, + output = "raw", + verbose = verbose ) %>% private$get_repo_url_from_response( type = type, @@ -99,10 +105,10 @@ GitHost <- R6::R6Class( ) } else if (!is.null(with_file)) { repo_urls <- private$get_repos_with_code( - code = with_file, - in_path = TRUE, - raw_output = TRUE, - verbose = verbose + code = with_file, + in_path = TRUE, + output = "raw", + verbose = verbose ) %>% private$get_repo_url_from_response( type = type, @@ -641,27 +647,27 @@ GitHost <- R6::R6Class( get_repos_with_code = function(code, in_files = NULL, in_path = FALSE, - raw_output = FALSE, + output = "table_full", verbose = TRUE, progress = TRUE) { if (private$scan_all) { repos_table <- private$get_repos_with_code_from_host( - code = code, - in_files = in_files, - in_path = in_path, - raw_output = raw_output, - verbose = verbose, - progress = progress + code = code, + in_files = in_files, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress ) } if (!private$scan_all) { repos_table <- private$get_repos_with_code_from_orgs( - code = code, - in_files = in_files, - in_path = in_path, - raw_output = raw_output, - verbose = verbose, - progress = progress + code = code, + in_files = in_files, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress ) } return(repos_table) @@ -702,11 +708,11 @@ GitHost <- R6::R6Class( # Pull repositories with code from whole Git Host get_repos_with_code_from_host = function(code, - in_files = NULL, - in_path = FALSE, - raw_output = FALSE, - verbose = TRUE, - progress = TRUE) { + in_files = NULL, + in_path = FALSE, + output = "table_full", + verbose = TRUE, + progress = TRUE) { if (verbose) { show_message( host = private$host_name, @@ -715,23 +721,29 @@ GitHost <- R6::R6Class( ) } repos_response <- private$get_repos_response_with_code( - code = code, - in_files = in_files, - in_path = in_path, - raw_output = raw_output, - verbose = verbose, - progress = progress + code = code, + in_files = in_files, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress ) - if (!raw_output) { + if (output != "raw") { rest_engine <- private$engines$rest repos_table <- repos_response %>% - rest_engine$tailor_repos_response() %>% + rest_engine$tailor_repos_response( + output = output + ) %>% rest_engine$prepare_repos_table( + output = output, verbose = verbose - ) %>% - rest_engine$get_repos_issues( - progress = progress ) + if (output == "table_full") { + repos_table <- repos_table %>% + rest_engine$get_repos_issues( + progress = progress + ) + } return(repos_table) } else { return(repos_response) @@ -740,11 +752,11 @@ GitHost <- R6::R6Class( # Pull repositories with code from given organizations get_repos_with_code_from_orgs = function(code, - in_files = NULL, - in_path = FALSE, - raw_output = FALSE, - verbose = TRUE, - progress = TRUE) { + in_files = NULL, + in_path = FALSE, + output = "table_full", + verbose = TRUE, + progress = TRUE) { repos_list <- purrr::map(private$orgs, function(org) { if (verbose) { show_message( @@ -755,31 +767,52 @@ GitHost <- R6::R6Class( information = "Pulling repositories" ) } - repos_response <- private$get_repos_response_with_code( - org = org, - code = code, - in_files = in_files, - in_path = in_path, - raw_output = raw_output, - verbose = verbose, - progress = progress - ) - if (!raw_output) { + rest_engine <- private$engines$rest + if (is.null(in_files)) { + repos_response <- rest_engine$get_repos_by_code( + org = org, + code = code, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress + ) + } else { + repos_response <- purrr::map(in_files, function(filename) { + rest_engine$get_repos_by_code( + org = org, + code = code, + filename = filename, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress + ) + }) %>% + purrr::list_flatten() + } + if (output != "raw") { rest_engine <- private$engines$rest repos_table <- repos_response %>% - rest_engine$tailor_repos_response() %>% + rest_engine$tailor_repos_response( + output = output + ) %>% rest_engine$prepare_repos_table( + output = output, verbose = verbose - ) %>% - rest_engine$get_repos_issues( - progress = progress ) + if (output == "table_full") { + repos_table <- repos_table %>% + rest_engine$get_repos_issues( + progress = progress + ) + } return(repos_table) } else { return(repos_response) } }, .progress = progress) - if (!raw_output) { + if (output != "raw") { repos_output <- purrr::list_rbind(repos_list) } else { repos_output <- purrr::list_flatten(repos_list) @@ -787,39 +820,13 @@ GitHost <- R6::R6Class( return(repos_output) }, - # Wrapper in case in_files is fed. - get_repos_response_with_code = function(org = NULL, - code, - in_files, - in_path, - raw_output, - verbose, - progress) { - rest_engine <- private$engines$rest - if (is.null(in_files)) { - repos_response <- rest_engine$get_repos_by_code( - org = org, - code = code, - in_path = in_path, - raw_output = raw_output, - verbose = verbose, - progress = progress + add_platform = function(repos_table) { + if (nrow(repos_table) > 0) { + dplyr::mutate( + repos_table, + platform = retrieve_platform(api_url) ) - } else { - repos_response <- purrr::map(in_files, function(filename) { - rest_engine$get_repos_by_code( - org = org, - code = code, - filename = filename, - in_path = in_path, - raw_output = raw_output, - verbose = verbose, - progress = progress - ) - }) %>% - purrr::list_flatten() } - return(repos_response) }, #' Add information on repository contributors. diff --git a/R/GitStats.R b/R/GitStats.R index 27ff5759..565e265d 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -449,23 +449,23 @@ GitStats <- R6::R6Class( }, #' @description Wrapper over pulling repositories by code. - #' @param package_name A character, name of the package. + #' @param packages A character vector, names of R packages to look for. #' @param only_loading A boolean, if `TRUE` function will check only if #' package is loaded in repositories, not used as dependencies. #' @param cache A logical, if set to `TRUE` GitStats will retrieve the last #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and #' printing output is switched off. - get_R_package_usage = function(package_name, + get_R_package_usage = function(packages, only_loading = FALSE, cache = TRUE, verbose = TRUE) { private$check_for_host() - if (is.null(package_name)) { - cli::cli_abort("You need to define `package_name`.", call = NULL) + if (is.null(packages)) { + cli::cli_abort("You need to define at least one `package_name`.", call = NULL) } args_list <- list( - "package_name" = package_name, + "packages" = packages, "only_loading" = only_loading ) trigger <- private$trigger_pulling( @@ -476,7 +476,7 @@ GitStats <- R6::R6Class( ) if (trigger) { R_package_usage <- private$get_R_package_usage_from_hosts( - package_name = package_name, + packages = packages, only_loading = only_loading, verbose = verbose ) @@ -487,14 +487,6 @@ GitStats <- R6::R6Class( attr_list = args_list ) private$save_to_storage(R_package_usage) - } else { - if (verbose) { - cli::cli_alert_warning( - cli::col_yellow( - "No usage of R packages found." - ) - ) - } } } else { R_package_usage <- private$get_from_storage( @@ -680,6 +672,7 @@ GitStats <- R6::R6Class( with_code, in_files = NULL, with_files, + output = "table_full", verbose = TRUE, progress = TRUE) { repos_table <- purrr::map(private$hosts, function(host) { @@ -689,6 +682,7 @@ GitStats <- R6::R6Class( add_contributors = add_contributors, with_code = with_code, in_files = in_files, + output = output, verbose = verbose, progress = progress ) @@ -697,6 +691,7 @@ GitStats <- R6::R6Class( host = host, add_contributors = add_contributors, with_files = with_files, + output = output, verbose = verbose, progress = progress ) @@ -709,8 +704,12 @@ GitStats <- R6::R6Class( } }) %>% purrr::list_rbind() %>% - private$add_stats_to_repos() %>% dplyr::as_tibble() + if (output == "table_full") { + repos_table <- repos_table %>% + private$add_stats_to_repos() %>% + dplyr::as_tibble() + } return(repos_table) }, @@ -719,6 +718,7 @@ GitStats <- R6::R6Class( add_contributors, with_code, in_files, + output, verbose, progress) { purrr::map(with_code, function(with_code) { @@ -726,6 +726,7 @@ GitStats <- R6::R6Class( add_contributors = add_contributors, with_code = with_code, in_files = in_files, + output = output, verbose = verbose, progress = progress ) @@ -737,12 +738,14 @@ GitStats <- R6::R6Class( get_repos_from_host_with_files = function(host, add_contributors, with_files, + output, verbose, progress) { purrr::map(with_files, function(with_file) { host$get_repos( add_contributors = add_contributors, with_file = with_file, + output = output, verbose = verbose, progress = progress ) @@ -951,36 +954,60 @@ GitStats <- R6::R6Class( }, # Pull information on package usage in a table form - get_R_package_usage_from_hosts = function(package_name, + get_R_package_usage_from_hosts = function(packages, only_loading, - verbose) { - if (!only_loading) { - repos_with_package_as_dependency <- private$get_R_package_as_dependency( + verbose = TRUE) { + packages_usage_tables <- purrr::map(packages, function(package_name) { + if (!only_loading) { + repos_with_package_as_dependency <- private$get_R_package_as_dependency( + package_name = package_name, + verbose = verbose + ) + } else { + repos_with_package_as_dependency <- NULL + } + repos_using_package <- private$get_R_package_loading( package_name = package_name, verbose = verbose ) - } else { - repos_with_package_as_dependency <- NULL - } - repos_using_package <- private$get_R_package_loading( - package_name = package_name, - verbose = verbose - ) - package_usage_table <- purrr::list_rbind( - list( - repos_with_package_as_dependency, - repos_using_package + package_usage_table <- purrr::list_rbind( + list( + repos_with_package_as_dependency, + repos_using_package + ) ) - ) - if (nrow(package_usage_table) > 0) { - duplicated_repos <- package_usage_table$api_url[duplicated(package_usage_table$api_url)] - package_usage_table <- package_usage_table[!duplicated(package_usage_table$api_url), ] - package_usage_table <- package_usage_table %>% - dplyr::mutate( - package_usage = ifelse(api_url %in% duplicated_repos, "import, library", package_usage) + if (nrow(package_usage_table) > 0) { + duplicated_repos <- package_usage_table$api_url[duplicated(package_usage_table$api_url)] + package_usage_table <- package_usage_table[!duplicated(package_usage_table$api_url), ] + package_usage_table <- package_usage_table %>% + dplyr::mutate( + package_usage = ifelse(api_url %in% duplicated_repos, "import, library", package_usage) + ) + package_usage_table <- dplyr::mutate( + package_usage_table, + package = package_name, + repo_fullname = paste0(organization, "/", repo_name) + ) %>% + dplyr::relocate( + package, package_usage, + .before = repo_id + ) %>% + dplyr::relocate( + repo_fullname, + .after = repo_id + ) + } + return(package_usage_table) + }) %>% + purrr::list_rbind() + if (nrow(packages_usage_tables) == 0 && verbose) { + cli::cli_alert_warning( + cli::col_yellow( + "No usage of R packages found." ) + ) } - return(package_usage_table) + return(packages_usage_tables) }, # Search repositories with `library(package_name)` in code blobs. @@ -994,14 +1021,13 @@ GitStats <- R6::R6Class( ) repos_using_package <- purrr::map(package_usage_phrases, ~ { repos_using_package <- private$get_repos_from_hosts( - with_code = ., - verbose = FALSE, - progress = FALSE + with_code = ., + output = "table_min", + verbose = FALSE, + progress = FALSE ) - if (!is.null(repos_using_package)) { + if (nrow(repos_using_package) > 0) { repos_using_package$package_usage <- "library" - repos_using_package <- repos_using_package %>% - dplyr::select(repo_name, organization, fullname, platform, repo_url, api_url, package_usage) } return(repos_using_package) }) %>% @@ -1017,16 +1043,15 @@ GitStats <- R6::R6Class( cli::cli_alert_info("Checking where [{package_name}] is used as a dependency...") } repos_with_package <- private$get_repos_from_hosts( - with_code = package_name, - in_files = c("DESCRIPTION", "NAMESPACE"), - verbose = FALSE, - progress = FALSE + with_code = package_name, + in_files = c("DESCRIPTION", "NAMESPACE"), + output = "table_min", + verbose = FALSE, + progress = FALSE ) - if (!is.null(repos_with_package)) { + if (nrow(repos_with_package) > 0) { repos_with_package <- repos_with_package[!duplicated(repos_with_package$api_url), ] repos_with_package$package_usage <- "import" - repos_with_package <- repos_with_package %>% - dplyr::select(repo_name, organization, fullname, platform, repo_url, api_url, package_usage) } return(repos_with_package) }, @@ -1043,8 +1068,7 @@ GitStats <- R6::R6Class( Sys.time(), last_activity_at, units = "days" - ) %>% round(2), - platform = retrieve_platform(api_url) + ) %>% round(2) ) %>% dplyr::relocate( organization, fullname, platform, repo_url, api_url, created_at, @@ -1207,13 +1231,13 @@ GitStats <- R6::R6Class( "commits" = "dates_range", "release_logs" = "dates_range", "users" = "logins", - "R_package_usage" = "package_name") + "R_package_usage" = "packages") attr_data <- attr(storage_table, storage_attr) attr_name <- switch(storage_attr, "file_path" = "files", "pattern" = "files matching pattern", "dates_range" = "date range", - "package_name" = "package", + "packages" = "packages", "logins" = "logins") if (length(attr_data) > 1) { separator <- if (storage_attr == "dates_range") { diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index a4cbd1c2..91a8f3a2 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -440,7 +440,7 @@ get_files_structure <- function(gitstats_object, #' loading package (`library(package)` and `require(package)` in all files) or #' using it as a dependency (`package` in `DESCRIPTION` and `NAMESPACE` files). #' @param gitstats_object A GitStats object. -#' @param package_name A character, name of the package. +#' @param packages A character vector, names of R packages to look for. #' @param only_loading A boolean, if `TRUE` function will check only if package #' is loaded in repositories, not used as dependencies. #' @param cache A logical, if set to `TRUE` GitStats will retrieve the last @@ -460,12 +460,12 @@ get_files_structure <- function(gitstats_object, #' } #' @export get_R_package_usage <- function(gitstats_object, - package_name, + packages, only_loading = FALSE, cache = TRUE, verbose = is_verbose(gitstats_object)) { gitstats_object$get_R_package_usage( - package_name = package_name, + packages = packages, only_loading = only_loading, cache = cache, verbose = verbose diff --git a/inst/package_usage_workflow.R b/inst/package_usage_workflow.R new file mode 100644 index 00000000..26b5288f --- /dev/null +++ b/inst/package_usage_workflow.R @@ -0,0 +1,13 @@ +devtools::load_all(".") + +test_gitstats <- create_gitstats() %>% + set_github_host( + orgs = "openpharma" + ) + +get_R_package_usage(test_gitstats, packages = "no_such_package") + +get_R_package_usage( + test_gitstats, + packages = c("purrr", "shiny") +) diff --git a/man/get_R_package_usage.Rd b/man/get_R_package_usage.Rd index 759fac09..3fce37da 100644 --- a/man/get_R_package_usage.Rd +++ b/man/get_R_package_usage.Rd @@ -6,7 +6,7 @@ \usage{ get_R_package_usage( gitstats_object, - package_name, + packages, only_loading = FALSE, cache = TRUE, verbose = is_verbose(gitstats_object) @@ -15,7 +15,7 @@ get_R_package_usage( \arguments{ \item{gitstats_object}{A GitStats object.} -\item{package_name}{A character, name of the package.} +\item{packages}{A character vector, names of R packages to look for.} \item{only_loading}{A boolean, if \code{TRUE} function will check only if package is loaded in repositories, not used as dependencies.} diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index 72bd89bc..9ca42bbd 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -13,6 +13,14 @@ Message i Preparing repositories table... +# `prepare_repos_table()` prepares minimum version of repos table + + Code + gh_repos_by_code_table_min <- test_rest_github$prepare_repos_table(repos_list = test_mocker$ + use("gh_repos_by_code_tailored_min"), output = "table_min") + Message + i Preparing repositories table... + # `get_all_repos()` works as expected Code @@ -20,9 +28,12 @@ Message i [Host:GitHub][Engine:GraphQl][Scope:test-org] Pulling repositories... -# `get_repos_contributors()` adds contributors to repos table +# `get_repos_contributors()` works on GitHost level Code - gh_repos_by_code_table <- test_rest_github$get_repos_contributors(repos_table = test_mocker$ - use("gh_repos_by_code_table"), progress = FALSE) + gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( + repos_table = test_mocker$use("gh_repos_table_with_platform"), verbose = TRUE, + progress = FALSE) + Message + i [Host:GitHub][Engine:REST] Pulling contributors... diff --git a/tests/testthat/_snaps/get_usage_R_package.md b/tests/testthat/_snaps/get_usage_R_package.md index 69eef51f..bf264385 100644 --- a/tests/testthat/_snaps/get_usage_R_package.md +++ b/tests/testthat/_snaps/get_usage_R_package.md @@ -1,8 +1,8 @@ # when get_R_package_usage_from_hosts output is empty return warning Code - test_gitstats$get_R_package_usage(package_name = "shiny", only_loading = FALSE, - verbose = TRUE) + test_gitstats$get_R_package_usage_from_hosts(packages = "non-existing-package", + only_loading = FALSE, verbose = TRUE) Message ! No usage of R packages found. Output diff --git a/tests/testthat/helper-expect-tables.R b/tests/testthat/helper-expect-tables.R index 65fa3c79..74b555b9 100644 --- a/tests/testthat/helper-expect-tables.R +++ b/tests/testthat/helper-expect-tables.R @@ -4,13 +4,18 @@ repo_gitstats_colnames <- c( "languages", "issues_open", "issues_closed" ) -repo_host_colnames <- c('repo_id', 'repo_name', 'default_branch', 'stars', 'forks', - 'created_at', 'last_activity_at', 'languages', 'issues_open', - 'issues_closed', 'organization', 'repo_url') +repo_host_colnames <- c("repo_id", "repo_name", "default_branch", "stars", "forks", + "created_at", "last_activity_at", "languages", "issues_open", + "issues_closed", "organization", "repo_url") + +repo_min_colnames <- c("repo_id", "repo_name", "default_branch", + "created_at", "organization", "repo_url") expect_package_usage_table <- function(object, with_cols = NULL) { expect_s3_class(object, "data.frame") - expect_named(object, c('repo_name', 'organization', 'fullname', 'platform', 'repo_url', 'api_url', 'package_usage')) + expect_named(object, c("package", "package_usage", "repo_id", "repo_fullname", + "repo_name", "default_branch", "created_at", "organization", + "repo_url", "api_url", "platform")) expect_gt(nrow(object), 0) } diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index c4bc6da6..4e1856af 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -154,7 +154,7 @@ test_that("`get_repos_by_code()` returns repos output for code search in files", test_mocker$cache(gh_repos_by_code) }) -test_that("`get_repos_by_code()` for GitHub prepares a raw (raw_output = TRUE) search response", { +test_that("`get_repos_by_code()` for GitHub prepares a raw search response", { mockery::stub( test_rest_github$get_repos_by_code, "self$response", @@ -171,10 +171,10 @@ test_that("`get_repos_by_code()` for GitHub prepares a raw (raw_output = TRUE) s test_mocker$use("gh_mapped_repos") ) gh_repos_by_code_raw <- test_rest_github$get_repos_by_code( - code = "test_code", - org = "test_org", - raw_output = TRUE, - verbose = FALSE + code = "test_code", + org = "test_org", + output = "raw", + verbose = FALSE ) expect_gh_search_response(gh_repos_by_code_raw$items) test_mocker$cache(gh_repos_by_code_raw) @@ -202,6 +202,29 @@ test_that("GitHub tailors precisely `repos_list`", { test_mocker$cache(gh_repos_by_code_tailored) }) +test_that("GitHub tailors `repos_list` to minimal version of table", { + gh_repos_by_code <- test_mocker$use("gh_repos_by_code") + gh_repos_by_code_tailored_min <- + test_rest_github$tailor_repos_response( + repos_response = gh_repos_by_code, + output = "table_min" + ) + gh_repos_by_code_tailored_min %>% + expect_type("list") %>% + expect_length(length(gh_repos_by_code)) + expect_list_contains_only( + gh_repos_by_code_tailored_min[[1]], + c( + "repo_id", "repo_name", "created_at", "organization" + ) + ) + expect_lt( + length(gh_repos_by_code_tailored_min[[1]]), + length(gh_repos_by_code[[1]]) + ) + test_mocker$cache(gh_repos_by_code_tailored_min) +}) + test_that("`prepare_repos_table()` prepares repos table", { expect_snapshot( gh_repos_by_code_table <- test_rest_github$prepare_repos_table( @@ -211,10 +234,23 @@ test_that("`prepare_repos_table()` prepares repos table", { expect_repos_table( gh_repos_by_code_table ) - gh_repos_by_code_table <- github_testhost_priv$add_repo_api_url(gh_repos_by_code_table) test_mocker$cache(gh_repos_by_code_table) }) +test_that("`prepare_repos_table()` prepares minimum version of repos table", { + expect_snapshot( + gh_repos_by_code_table_min <- test_rest_github$prepare_repos_table( + repos_list = test_mocker$use("gh_repos_by_code_tailored_min"), + output = "table_min" + ) + ) + expect_repos_table( + gh_repos_by_code_table_min, + repo_cols = repo_min_colnames + ) + test_mocker$cache(gh_repos_by_code_table_min) +}) + test_that("`get_repos_issues()` adds issues to repos table", { mockery::stub( test_rest_github$get_repos_issues, @@ -242,7 +278,7 @@ test_that("`get_repos_issues()` adds issues to repos table", { test_that("`get_repos_with_code_from_orgs()` works", { mockery::stub( github_testhost_priv$get_repos_with_code_from_orgs, - "private$get_repos_response_with_code", + "rest_engine$get_repos_by_code", test_mocker$use("gh_repos_by_code") ) mockery::stub( @@ -250,11 +286,84 @@ test_that("`get_repos_with_code_from_orgs()` works", { "rest_engine$get_repos_issues", test_mocker$use("gh_repos_by_code_table") ) - repos_with_code <- github_testhost_priv$get_repos_with_code_from_orgs( + repos_with_code_from_orgs_full <- github_testhost_priv$get_repos_with_code_from_orgs( + code = "shiny", + output = "table_full", + verbose = FALSE + ) + expect_repos_table(repos_with_code_from_orgs_full) + test_mocker$cache(repos_with_code_from_orgs_full) +}) + +test_that("`get_repos_with_code_from_orgs()` pulls minimum version of table", { + mockery::stub( + github_testhost_priv$get_repos_with_code_from_orgs, + "rest_engine$get_repos_by_code", + test_mocker$use("gh_repos_by_code") + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_orgs, + "rest_engine$prepare_repos_table", + test_mocker$use("gh_repos_by_code_table_min") + ) + repos_with_code_from_orgs_min <- github_testhost_priv$get_repos_with_code_from_orgs( code = "shiny", + output = "table_min", verbose = FALSE ) - expect_repos_table(repos_with_code, with_cols = "api_url") + expect_repos_table(repos_with_code_from_orgs_min, + repo_cols = repo_min_colnames) + test_mocker$cache(repos_with_code_from_orgs_min) +}) + +test_that("`get_repos_with_code_from_orgs()` pulls raw response", { + mockery::stub( + github_testhost_priv$get_repos_with_code_from_orgs, + "rest_engine$get_repos_by_code", + test_mocker$use("gh_repos_by_code_raw") + ) + repos_with_code_from_orgs_raw <- github_testhost_priv$get_repos_with_code_from_orgs( + code = "shiny", + output = "raw", + verbose = FALSE + ) + expect_type(repos_with_code_from_orgs_raw, "list") + expect_gt(length(repos_with_code_from_orgs_raw), 0) +}) + +test_that("get_repos_with_code() works", { + mockery::stub( + github_testhost_priv$get_repos_with_code, + "private$get_repos_with_code_from_orgs", + test_mocker$use("repos_with_code_from_orgs_full") + ) + github_repos_with_code <- github_testhost_priv$get_repos_with_code( + code = "test-code", + verbose = FALSE, + progress = FALSE + ) + expect_repos_table( + github_repos_with_code + ) +}) + +test_that("get_repos_with_code() works", { + mockery::stub( + github_testhost_priv$get_repos_with_code, + "private$get_repos_with_code_from_orgs", + test_mocker$use("repos_with_code_from_orgs_min") + ) + github_repos_with_code_min <- github_testhost_priv$get_repos_with_code( + code = "test-code", + output = "table_min", + verbose = FALSE, + progress = FALSE + ) + expect_repos_table( + github_repos_with_code_min, + repo_cols = repo_min_colnames + ) + test_mocker$cache(github_repos_with_code_min) }) test_that("GitHub prepares repos table from repositories response", { @@ -267,29 +376,40 @@ test_that("GitHub prepares repos table from repositories response", { test_mocker$cache(gh_repos_table) }) -test_that("GitHost adds `repo_api_url` column to GitHub repos table", { - repos_table <- test_mocker$use("gh_repos_table") - gh_repos_table_with_api_url <- github_testhost_priv$add_repo_api_url(repos_table) - expect_true(all(grepl("api.github.com", gh_repos_table_with_api_url$api_url))) - test_mocker$cache(gh_repos_table_with_api_url) -}) - test_that("`get_all_repos()` works as expected", { mockery::stub( github_testhost_priv$get_all_repos, "graphql_engine$prepare_repos_table", - test_mocker$use("gh_repos_table_with_api_url") + test_mocker$use("gh_repos_table") ) expect_snapshot( gh_repos_table <- github_testhost_priv$get_all_repos() ) expect_repos_table( - gh_repos_table, - with_cols = "api_url" + gh_repos_table ) test_mocker$cache(gh_repos_table) }) +test_that("GitHost adds `repo_api_url` column to GitHub repos table", { + repos_table <- test_mocker$use("gh_repos_table") + gh_repos_table_with_api_url <- github_testhost_priv$add_repo_api_url(repos_table) + expect_true(all(grepl("api.github.com", gh_repos_table_with_api_url$api_url))) + test_mocker$cache(gh_repos_table_with_api_url) +}) + +test_that("add_platform adds data on Git platform to repos table", { + gh_repos_table_with_platform <- github_testhost_priv$add_platform( + repos_table = test_mocker$use("gh_repos_table_with_api_url") + ) + expect_repos_table( + gh_repos_table_with_platform, + with_cols = c("api_url", "platform") + ) + test_mocker$cache(gh_repos_table_with_platform) +}) + + test_that("get_contributors_from_repo", { mockery::stub( test_rest_github_priv$get_contributors_from_repo, @@ -311,19 +431,100 @@ test_that("`get_repos_contributors()` adds contributors to repos table", { "private$get_contributors_from_repo", test_mocker$use("github_contributors") ) + gh_repos_with_contributors <- test_rest_github$get_repos_contributors( + repos_table = test_mocker$use("gh_repos_table_with_platform"), + progress = FALSE + ) + expect_repos_table( + gh_repos_with_contributors, + with_cols = c("api_url", "platform", "contributors") + ) + expect_gt( + length(gh_repos_with_contributors$contributors), + 0 + ) + test_mocker$cache(gh_repos_with_contributors) +}) + +test_that("`get_repos_contributors()` works on GitHost level", { + mockery::stub( + github_testhost_priv$get_repos_contributors, + "rest_engine$get_repos_contributors", + test_mocker$use("gh_repos_with_contributors") + ) expect_snapshot( - gh_repos_by_code_table <- test_rest_github$get_repos_contributors( - repos_table = test_mocker$use("gh_repos_by_code_table"), + gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( + repos_table = test_mocker$use("gh_repos_table_with_platform"), + verbose = TRUE, progress = FALSE ) ) expect_repos_table( - gh_repos_by_code_table, - with_cols = c("api_url", "contributors") + gh_repos_with_contributors, + with_cols = c("api_url", "platform", "contributors") ) expect_gt( - length(gh_repos_by_code_table$contributors), + length(gh_repos_with_contributors$contributors), 0 ) - test_mocker$cache(gh_repos_by_code_table) + test_mocker$cache(gh_repos_with_contributors) +}) + +test_that("`get_repos()` works as expected", { + mockery::stub( + github_testhost$get_repos, + "private$get_all_repos", + test_mocker$use("gh_repos_table") + ) + gh_repos_table <- github_testhost$get_repos( + add_contributors = FALSE, + verbose = FALSE + ) + expect_repos_table( + gh_repos_table, + with_cols = c("api_url", "platform") + ) + test_mocker$cache(gh_repos_table) +}) + +test_that("`get_repos()` works as expected", { + mockery::stub( + github_testhost$get_repos, + "private$get_all_repos", + test_mocker$use("gh_repos_table") + ) + mockery::stub( + github_testhost$get_repos, + "private$get_repos_contributors", + test_mocker$use("gh_repos_with_contributors") + ) + gh_repos_table_full <- github_testhost$get_repos( + add_contributors = TRUE, + verbose = FALSE + ) + expect_repos_table( + gh_repos_table_full, + with_cols = c("api_url", "platform", "contributors") + ) + test_mocker$cache(gh_repos_table_full) +}) + +test_that("`get_repos()` pulls table in minimalist version", { + mockery::stub( + github_testhost$get_repos, + "private$get_repos_with_code", + test_mocker$use("github_repos_with_code_min") + ) + gh_repos_table_min <- github_testhost$get_repos( + add_contributors = FALSE, + with_code = "test_code", + output = "table_min", + verbose = FALSE + ) + expect_repos_table( + gh_repos_table_min, + repo_cols = repo_min_colnames, + with_cols = c("api_url", "platform") + ) + test_mocker$cache(gh_repos_table_min) }) diff --git a/tests/testthat/test-01-get_repos-GitStats.R b/tests/testthat/test-01-get_repos-GitStats.R index 10fab878..20d4f34c 100644 --- a/tests/testthat/test-01-get_repos-GitStats.R +++ b/tests/testthat/test-01-get_repos-GitStats.R @@ -3,11 +3,12 @@ test_that("get_repos_from_hosts works", { test_gitstats_priv$get_repos_from_hosts, "host$get_repos", purrr::list_rbind(list( - test_mocker$use("gh_repos_table_with_api_url"), + test_mocker$use("gh_repos_table_full"), test_mocker$use("gl_repos_table_with_api_url") )) ) - repos_table <- test_gitstats_priv$get_repos_from_hosts( + repos_from_hosts <- test_gitstats_priv$get_repos_from_hosts( + add_contributors = TRUE, with_code = NULL, in_files = NULL, with_files = NULL, @@ -15,38 +16,39 @@ test_that("get_repos_from_hosts works", { progress = FALSE ) expect_repos_table( - repos_table, - repo_cols = repo_gitstats_colnames + repos_from_hosts, + repo_cols = repo_gitstats_colnames, + with_cols = c("contributors", "contributors_n") ) + test_mocker$cache(repos_from_hosts) }) -test_that("get_repos_from_hosts with_code works", { +test_that("get_repos_from_hosts pulls table in minimalist version", { mockery::stub( test_gitstats_priv$get_repos_from_hosts, - "private$get_repos_from_host_with_code", - purrr::list_rbind( - list(test_mocker$use("gh_repos_by_code_table"), - test_mocker$use("gl_repos_by_code_table")) - ) + "host$get_repos", + test_mocker$use("gh_repos_table_min") ) - repos_table <- test_gitstats_priv$get_repos_from_hosts( - with_code = "shiny", - in_files = "DESCRIPTION", + repos_from_hosts_min <- test_gitstats_priv$get_repos_from_hosts( + add_contributors = TRUE, + with_code = NULL, + in_files = NULL, with_files = NULL, - verbose = FALSE, - progress = FALSE + output = "table_min", + verbose = FALSE, + progress = FALSE ) expect_repos_table( - repos_table, - repo_cols = repo_gitstats_colnames, - with_cols = c("contributors", "contributors_n") + repos_from_hosts_min, + repo_cols = repo_min_colnames, + with_cols = c("api_url", "platform") ) - test_mocker$cache(repos_table) + test_mocker$cache(repos_from_hosts_min) }) test_that("set_object_class for repos_table works correctly", { repos_table <- test_gitstats_priv$set_object_class( - object = test_mocker$use("repos_table"), + object = test_mocker$use("repos_from_hosts"), class = "repos_table", attr_list = list( "with_code" = NULL, diff --git a/tests/testthat/test-get_usage_R_package.R b/tests/testthat/test-get_usage_R_package.R index 025c923f..71b4e74d 100644 --- a/tests/testthat/test-get_usage_R_package.R +++ b/tests/testthat/test-get_usage_R_package.R @@ -2,7 +2,7 @@ test_that("get_R_package_as_dependency work correctly", { mockery::stub( test_gitstats_priv$get_R_package_as_dependency, "private$get_repos_from_hosts", - test_mocker$use("repos_table") + test_mocker$use("repos_from_hosts_min") ) R_package_as_dependency <- test_gitstats_priv$get_R_package_as_dependency( package_name = "shiny", @@ -19,6 +19,27 @@ test_that("get_R_package_as_dependency work correctly", { test_mocker$cache(R_package_as_dependency) }) +test_that("get_R_package_loading work correctly", { + mockery::stub( + test_gitstats_priv$get_R_package_loading, + "private$get_repos_from_hosts", + test_mocker$use("repos_from_hosts_min") + ) + R_package_loading <- test_gitstats_priv$get_R_package_loading( + package_name = "purrr", + verbose = FALSE + ) + expect_s3_class( + R_package_loading, + "data.frame" + ) + expect_gt( + nrow(R_package_loading), + 0 + ) + test_mocker$cache(R_package_loading) +}) + test_that("get_R_package_usage_from_hosts works as expected", { test_gitstats <- create_test_gitstats(hosts = 2, priv_mode = TRUE) mockery::stub( @@ -29,25 +50,51 @@ test_that("get_R_package_usage_from_hosts works as expected", { mockery::stub( test_gitstats$get_R_package_usage_from_hosts, "private$get_R_package_loading", - test_mocker$use("R_package_as_dependency") + test_mocker$use("R_package_loading") ) R_package_usage_table <- test_gitstats$get_R_package_usage_from_hosts( - package_name = "shiny", only_loading = FALSE, verbose = FALSE + packages = c("shiny", "purrr"), + only_loading = FALSE, + verbose = FALSE ) expect_package_usage_table(R_package_usage_table) test_mocker$cache(R_package_usage_table) }) test_that("when get_R_package_usage_from_hosts output is empty return warning", { - test_gitstats <- create_test_gitstats(hosts = 2) + test_gitstats <- create_test_gitstats(hosts = 2, priv_mode = TRUE) mockery::stub( - test_gitstats$get_R_package_usage, - "private$get_R_package_usage_from_hosts", + test_gitstats$get_R_package_usage_from_hosts, + "private$get_R_package_loading", + data.frame() + ) + mockery::stub( + test_gitstats$get_R_package_usage_from_hosts, + "private$get_R_package_as_dependency", data.frame() ) expect_snapshot( - test_gitstats$get_R_package_usage( - package_name = "shiny", only_loading = FALSE, verbose = TRUE + test_gitstats$get_R_package_usage_from_hosts( + packages = "non-existing-package", + only_loading = FALSE, + verbose = TRUE ) ) }) + +test_that("get_R_package_usage works", { + mockery::stub( + test_gitstats$get_R_package_usage, + "private$get_R_package_usage_from_hosts", + test_mocker$use("R_package_usage_table") + ) + R_package_usage_table <- test_gitstats$get_R_package_usage( + packages = c("shiny", "purrr"), + verbose = FALSE + ) + expect_package_usage_table(R_package_usage_table) + expect_s3_class( + R_package_usage_table, + "R_package_usage" + ) +}) diff --git a/vignettes/get_repos_with_code.Rmd b/vignettes/get_repos_with_code.Rmd index 65c28e0b..163223bf 100644 --- a/vignettes/get_repos_with_code.Rmd +++ b/vignettes/get_repos_with_code.Rmd @@ -56,11 +56,11 @@ repos_urls <- get_repos_urls( ## Package usage -`GitStats` allows you to search for repositories which make use of certain R packages. This function scans repositories if they import packages (look for package name in `DESCRIPTION` or `NAMESPACE`) or if the load package with `library()` or `request()`. +`GitStats` allows you to search for repositories which make use of certain R packages. This function scans repositories if they import packages (look for package name in `DESCRIPTION` or `NAMESPACE`) or if they load package with `library()` or `request()`. ```{r, eval = FALSE} package_usage <- get_R_package_usage( gitstats_object = github_stats, - package_name = "shiny" + packages = c("shiny", "purrr") ) ``` From 8484f15b39092688619843ce1703401ed0add778 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 17 Oct 2024 08:28:23 +0000 Subject: [PATCH 53/94] Add split_output parameter, update example in docs. --- NEWS.md | 1 + R/GitStats.R | 39 +++++++++++++++++------ R/gitstats_functions.R | 19 ++++++++--- inst/package_usage_workflow.R | 6 ++++ man/get_R_package_usage.Rd | 19 ++++++++--- tests/testthat/test-get_usage_R_package.R | 22 +++++++++++++ vignettes/get_repos_with_code.Rmd | 3 +- 7 files changed, 90 insertions(+), 19 deletions(-) diff --git a/NEWS.md b/NEWS.md index 911cb248..e2f3fc3e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ - Optimized `get_R_package_usage()` function: - you can now pass a vector of packages names (new `packages` parameter replacing old `package_name`) ([#494](https://github.com/r-world-devs/GitStats/issues/494)), - on the other hand, output of the function has been limited to contain only most necessary data (removing all repository stats), making thus process of obtaining package usage faster ([#474](https://github.com/r-world-devs/GitStats/issues/474)). + - new `split_output` parameter has been added - when set to `TRUE` a list with tibbles (every element of the list for every package) instead of one tibble is returned. - Added possibility to get repositories for individual users with `get_repos()` ([#492](https://github.com/r-world-devs/GitStats/issues/492)). Earlier this was only possible for GitHub organizations and GitLab groups. ## Fixes: diff --git a/R/GitStats.R b/R/GitStats.R index 565e265d..ca14f603 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -452,12 +452,17 @@ GitStats <- R6::R6Class( #' @param packages A character vector, names of R packages to look for. #' @param only_loading A boolean, if `TRUE` function will check only if #' package is loaded in repositories, not used as dependencies. + #' @param split_output Optional, a boolean. If `TRUE` will return a list of + #' tables, where every element of the list stands for the package passed to + #' `packages` parameter. If `FALSE`, will return only one table with name of + #' the package stored in first column. #' @param cache A logical, if set to `TRUE` GitStats will retrieve the last #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and #' printing output is switched off. get_R_package_usage = function(packages, only_loading = FALSE, + split_output = FALSE, cache = TRUE, verbose = TRUE) { private$check_for_host() @@ -478,9 +483,11 @@ GitStats <- R6::R6Class( R_package_usage <- private$get_R_package_usage_from_hosts( packages = packages, only_loading = only_loading, + split_output = split_output, verbose = verbose ) - if (nrow(R_package_usage) > 0) { + if ((!split_output && nrow(R_package_usage) > 0) || + (split_output && any(purrr::map_lgl(R_package_usage, ~ nrow(.) > 0)))) { R_package_usage <- private$set_object_class( object = R_package_usage, class = "R_package_usage", @@ -956,8 +963,9 @@ GitStats <- R6::R6Class( # Pull information on package usage in a table form get_R_package_usage_from_hosts = function(packages, only_loading, + split_output = FALSE, verbose = TRUE) { - packages_usage_tables <- purrr::map(packages, function(package_name) { + packages_usage_list <- purrr::map(packages, function(package_name) { if (!only_loading) { repos_with_package_as_dependency <- private$get_R_package_as_dependency( package_name = package_name, @@ -998,16 +1006,27 @@ GitStats <- R6::R6Class( ) } return(package_usage_table) - }) %>% - purrr::list_rbind() - if (nrow(packages_usage_tables) == 0 && verbose) { - cli::cli_alert_warning( - cli::col_yellow( - "No usage of R packages found." + }) + if (split_output) { + packages_usage_result <- purrr::set_names(packages_usage_list, packages) + if (all(purrr::map_lgl(packages_usage_result, ~ nrow(.) == 0)) && verbose) { + cli::cli_alert_warning( + cli::col_yellow( + "No usage of R packages found." + ) ) - ) + } + } else { + packages_usage_result <- purrr::list_rbind(packages_usage_list) + if (nrow(packages_usage_result) == 0 && verbose) { + cli::cli_alert_warning( + cli::col_yellow( + "No usage of R packages found." + ) + ) + } } - return(packages_usage_tables) + return(packages_usage_result) }, # Search repositories with `library(package_name)` in code blobs. diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index 91a8f3a2..f693041b 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -438,15 +438,20 @@ get_files_structure <- function(gitstats_object, #' @name get_R_package_usage #' @description Wrapper over searching repositories by code blobs related to #' loading package (`library(package)` and `require(package)` in all files) or -#' using it as a dependency (`package` in `DESCRIPTION` and `NAMESPACE` files). +#' using it as a dependency (`package` in `DESCRIPTION` and `NAMESPACE` +#' files). #' @param gitstats_object A GitStats object. #' @param packages A character vector, names of R packages to look for. #' @param only_loading A boolean, if `TRUE` function will check only if package #' is loaded in repositories, not used as dependencies. +#' @param split_output Optional, a boolean. If `TRUE` will return a list of +#' tables, where every element of the list stands for the package passed to +#' `packages` parameter. If `FALSE`, will return only one table with name of +#' the package stored in first column. #' @param cache A logical, if set to `TRUE` GitStats will retrieve the last #' result from its storage. -#' @param verbose A logical, `TRUE` by default. If `FALSE` messages and -#' printing output is switched off. +#' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing +#' output is switched off. #' @return A data.frame. #' @examples #' \dontrun{ @@ -456,17 +461,23 @@ get_files_structure <- function(gitstats_object, #' orgs = c("r-world-devs", "openpharma") #' ) #' -#' get_R_package_usage(my_gitstats, "Shiny") +#' get_R_package_usage( +#' gitstats_object = my_gitstats, +#' packages = c("purrr", "shiny"), +#' split_output = TRUE +#' ) #' } #' @export get_R_package_usage <- function(gitstats_object, packages, only_loading = FALSE, + split_output = FALSE, cache = TRUE, verbose = is_verbose(gitstats_object)) { gitstats_object$get_R_package_usage( packages = packages, only_loading = only_loading, + split_output = split_output, cache = cache, verbose = verbose ) diff --git a/inst/package_usage_workflow.R b/inst/package_usage_workflow.R index 26b5288f..d7cf27e8 100644 --- a/inst/package_usage_workflow.R +++ b/inst/package_usage_workflow.R @@ -11,3 +11,9 @@ get_R_package_usage( test_gitstats, packages = c("purrr", "shiny") ) + +get_R_package_usage( + test_gitstats, + packages = c("dplyr", "shiny"), + split_output = TRUE +) diff --git a/man/get_R_package_usage.Rd b/man/get_R_package_usage.Rd index 3fce37da..e19087b1 100644 --- a/man/get_R_package_usage.Rd +++ b/man/get_R_package_usage.Rd @@ -8,6 +8,7 @@ get_R_package_usage( gitstats_object, packages, only_loading = FALSE, + split_output = FALSE, cache = TRUE, verbose = is_verbose(gitstats_object) ) @@ -20,11 +21,16 @@ get_R_package_usage( \item{only_loading}{A boolean, if \code{TRUE} function will check only if package is loaded in repositories, not used as dependencies.} +\item{split_output}{Optional, a boolean. If \code{TRUE} will return a list of +tables, where every element of the list stands for the package passed to +\code{packages} parameter. If \code{FALSE}, will return only one table with name of +the package stored in first column.} + \item{cache}{A logical, if set to \code{TRUE} GitStats will retrieve the last result from its storage.} -\item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and -printing output is switched off.} +\item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and printing +output is switched off.} } \value{ A data.frame. @@ -32,7 +38,8 @@ A data.frame. \description{ Wrapper over searching repositories by code blobs related to loading package (\code{library(package)} and \code{require(package)} in all files) or -using it as a dependency (\code{package} in \code{DESCRIPTION} and \code{NAMESPACE} files). +using it as a dependency (\code{package} in \code{DESCRIPTION} and \code{NAMESPACE} +files). } \examples{ \dontrun{ @@ -42,6 +49,10 @@ using it as a dependency (\code{package} in \code{DESCRIPTION} and \code{NAMESPA orgs = c("r-world-devs", "openpharma") ) - get_R_package_usage(my_gitstats, "Shiny") + get_R_package_usage( + gitstats_object = my_gitstats, + packages = c("purrr", "shiny"), + split_output = TRUE + ) } } diff --git a/tests/testthat/test-get_usage_R_package.R b/tests/testthat/test-get_usage_R_package.R index 71b4e74d..050cd2de 100644 --- a/tests/testthat/test-get_usage_R_package.R +++ b/tests/testthat/test-get_usage_R_package.R @@ -61,6 +61,28 @@ test_that("get_R_package_usage_from_hosts works as expected", { test_mocker$cache(R_package_usage_table) }) +test_that("get_R_package_usage_from_hosts with split_output works", { + test_gitstats <- create_test_gitstats(hosts = 2, priv_mode = TRUE) + mockery::stub( + test_gitstats$get_R_package_usage_from_hosts, + "private$get_R_package_as_dependency", + test_mocker$use("R_package_as_dependency") + ) + mockery::stub( + test_gitstats$get_R_package_usage_from_hosts, + "private$get_R_package_loading", + test_mocker$use("R_package_loading") + ) + R_package_usage_list <- test_gitstats$get_R_package_usage_from_hosts( + packages = c("shiny", "purrr"), + only_loading = FALSE, + split_output = TRUE, + verbose = FALSE + ) + expect_equal(names(R_package_usage_list), c("shiny", "purrr")) + purrr::walk(R_package_usage_list, expect_package_usage_table) +}) + test_that("when get_R_package_usage_from_hosts output is empty return warning", { test_gitstats <- create_test_gitstats(hosts = 2, priv_mode = TRUE) mockery::stub( diff --git a/vignettes/get_repos_with_code.Rmd b/vignettes/get_repos_with_code.Rmd index 163223bf..71c58a26 100644 --- a/vignettes/get_repos_with_code.Rmd +++ b/vignettes/get_repos_with_code.Rmd @@ -61,6 +61,7 @@ repos_urls <- get_repos_urls( ```{r, eval = FALSE} package_usage <- get_R_package_usage( gitstats_object = github_stats, - packages = c("shiny", "purrr") + packages = c("shiny", "purrr"), + split_output = TRUE ) ``` From b257b8bcc38f374099cdbb2d9a67ecf6440e7335 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 17 Oct 2024 09:02:52 +0000 Subject: [PATCH 54/94] Adjust and test getting repos for whole hosts. --- R/GitHost.R | 32 ++++++++++++++++------- tests/testthat/test-01-get_repos-GitHub.R | 17 ++++++++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index 4717d5d2..9d7015e4 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -720,16 +720,29 @@ GitHost <- R6::R6Class( information = "Pulling repositories" ) } - repos_response <- private$get_repos_response_with_code( - code = code, - in_files = in_files, - in_path = in_path, - output = output, - verbose = verbose, - progress = progress - ) + rest_engine <- private$engines$rest + if (is.null(in_files)) { + repos_response <- rest_engine$get_repos_by_code( + code = code, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress + ) + } else { + repos_response <- purrr::map(in_files, function(filename) { + rest_engine$get_repos_by_code( + code = code, + filename = filename, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress + ) + }) %>% + purrr::list_flatten() + } if (output != "raw") { - rest_engine <- private$engines$rest repos_table <- repos_response %>% rest_engine$tailor_repos_response( output = output @@ -792,7 +805,6 @@ GitHost <- R6::R6Class( purrr::list_flatten() } if (output != "raw") { - rest_engine <- private$engines$rest repos_table <- repos_response %>% rest_engine$tailor_repos_response( output = output diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 4e1856af..6111ee36 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -324,6 +324,7 @@ test_that("`get_repos_with_code_from_orgs()` pulls raw response", { ) repos_with_code_from_orgs_raw <- github_testhost_priv$get_repos_with_code_from_orgs( code = "shiny", + in_files = c("DESCRIPTION", "NAMESPACE"), output = "raw", verbose = FALSE ) @@ -331,6 +332,22 @@ test_that("`get_repos_with_code_from_orgs()` pulls raw response", { expect_gt(length(repos_with_code_from_orgs_raw), 0) }) +test_that("`get_repos_with_code_from_host()` pulls raw response", { + mockery::stub( + github_testhost_priv$get_repos_with_code_from_host, + "rest_engine$get_repos_by_code", + test_mocker$use("gh_repos_by_code_raw") + ) + repos_with_code_from_host_raw <- github_testhost_priv$get_repos_with_code_from_host( + code = "shiny", + in_files = c("DESCRIPTION", "NAMESPACE"), + output = "raw", + verbose = FALSE + ) + expect_type(repos_with_code_from_host_raw, "list") + expect_gt(length(repos_with_code_from_host_raw), 0) +}) + test_that("get_repos_with_code() works", { mockery::stub( github_testhost_priv$get_repos_with_code, From 1dc7a055972f269267fc4723aea5e343dc876b97 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 17 Oct 2024 10:19:36 +0000 Subject: [PATCH 55/94] Fix spellcheck. --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index e2f3fc3e..96b57bae 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,7 +5,7 @@ - Optimized `get_R_package_usage()` function: - you can now pass a vector of packages names (new `packages` parameter replacing old `package_name`) ([#494](https://github.com/r-world-devs/GitStats/issues/494)), - on the other hand, output of the function has been limited to contain only most necessary data (removing all repository stats), making thus process of obtaining package usage faster ([#474](https://github.com/r-world-devs/GitStats/issues/474)). - - new `split_output` parameter has been added - when set to `TRUE` a list with tibbles (every element of the list for every package) instead of one tibble is returned. + - new `split_output` parameter has been added - when set to `TRUE` a `list` with `tibbles` (every element of the `list` for every package) instead of one `tibble` is returned. - Added possibility to get repositories for individual users with `get_repos()` ([#492](https://github.com/r-world-devs/GitStats/issues/492)). Earlier this was only possible for GitHub organizations and GitLab groups. ## Fixes: From 498eb4e0a46f7514da6b011b0a73649456fb9c46 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 17 Oct 2024 10:22:51 +0000 Subject: [PATCH 56/94] Update docs. --- R/gitstats_functions.R | 3 ++- man/get_R_package_usage.Rd | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index f693041b..54c16d07 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -452,7 +452,8 @@ get_files_structure <- function(gitstats_object, #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing #' output is switched off. -#' @return A data.frame. +#' @return A `tibble` or `list` of `tibbles` depending on `split_output` +#' parameter. #' @examples #' \dontrun{ #' my_gitstats <- create_gitstats() %>% diff --git a/man/get_R_package_usage.Rd b/man/get_R_package_usage.Rd index e19087b1..9e56c0e5 100644 --- a/man/get_R_package_usage.Rd +++ b/man/get_R_package_usage.Rd @@ -33,7 +33,8 @@ result from its storage.} output is switched off.} } \value{ -A data.frame. +A \code{tibble} or \code{list} of \code{tibbles} depending on \code{split_output} +parameter. } \description{ Wrapper over searching repositories by code blobs related to From 4c406fc4e93b1fe29cdf86dd12cf9b8982af6010 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 17 Oct 2024 12:46:59 +0000 Subject: [PATCH 57/94] Fix printing storage for commits, release logs (objects with date ranges) and for repos URLs. --- R/GitStats.R | 26 +++++++++--------- inst/get_commits_workflow.R | 30 ++++++++++++++++++++ inst/get_repos_urls_workflow.R | 8 ++++++ tests/testthat/test-get_commits-GitStats.R | 32 +++++++++++++++++----- 4 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 inst/get_commits_workflow.R create mode 100644 inst/get_repos_urls_workflow.R diff --git a/R/GitStats.R b/R/GitStats.R index ca14f603..2faccf64 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -207,8 +207,7 @@ GitStats <- R6::R6Class( verbose = TRUE, progress = TRUE) { private$check_for_host() - args_list <- list("since" = since, - "until" = until) + args_list <- list("date_range" = c(since, until)) trigger <- private$trigger_pulling( cache = cache, storage = "commits", @@ -226,7 +225,9 @@ GitStats <- R6::R6Class( class = "commits_data", attr_list = args_list ) - private$save_to_storage(commits) + private$save_to_storage( + table = commits + ) } else { commits <- private$get_from_storage( table = "commits", @@ -417,10 +418,7 @@ GitStats <- R6::R6Class( verbose = TRUE, progress = TRUE) { private$check_for_host() - args_list <- list( - "since" = since, - "until" = until - ) + args_list <- list("date_range" = c(since, until)) trigger <- private$trigger_pulling( storage = "release_logs", cache = cache, @@ -1242,24 +1240,26 @@ GitStats <- R6::R6Class( }, # print storage attribute - print_storage_attribute = function(storage_table, storage_name) { + print_storage_attribute = function(storage_data, storage_name) { if (storage_name != "repositories") { storage_attr <- switch(storage_name, + "repos_urls" = "type", "files" = "file_path", "files_structure" = "pattern", - "commits" = "dates_range", - "release_logs" = "dates_range", + "commits" = "date_range", + "release_logs" = "date_range", "users" = "logins", "R_package_usage" = "packages") - attr_data <- attr(storage_table, storage_attr) + attr_data <- attr(storage_data, storage_attr) attr_name <- switch(storage_attr, + "type" = "type", "file_path" = "files", "pattern" = "files matching pattern", - "dates_range" = "date range", + "date_range" = "date range", "packages" = "packages", "logins" = "logins") if (length(attr_data) > 1) { - separator <- if (storage_attr == "dates_range") { + separator <- if (storage_attr == "date_range") { " - " } else { ", " diff --git a/inst/get_commits_workflow.R b/inst/get_commits_workflow.R new file mode 100644 index 00000000..df7fd15c --- /dev/null +++ b/inst/get_commits_workflow.R @@ -0,0 +1,30 @@ +devtools::load_all(".") + +git_stats <- create_gitstats() %>% + set_github_host( + orgs = c("r-world-devs", "openpharma"), + token = Sys.getenv("GITHUB_PAT") + ) %>% + set_gitlab_host( + orgs = c("mbtests"), + token = Sys.getenv("GITLAB_PAT_PUBLIC") + ) + +release_logs <- get_release_logs( + gitstats_object = git_stats, + since = "2024-01-01", + verbose = FALSE +) + +release_logs + +# Check printing in storage +test_gitstats + +get_commits( + git_stats, + since = "2024-01-01" +) + +# Check printing in storage +test_gitstats diff --git a/inst/get_repos_urls_workflow.R b/inst/get_repos_urls_workflow.R new file mode 100644 index 00000000..ad357185 --- /dev/null +++ b/inst/get_repos_urls_workflow.R @@ -0,0 +1,8 @@ +devtools::load_all(".") + +test_gitstats <- create_gitstats() |> + set_github_host( + orgs = c("r-world-devs", "openpharma") + ) + +get_repos_urls(test_gitstats, with_code = "shiny") diff --git a/tests/testthat/test-get_commits-GitStats.R b/tests/testthat/test-get_commits-GitStats.R index 0a3fa461..aa23335f 100644 --- a/tests/testthat/test-get_commits-GitStats.R +++ b/tests/testthat/test-get_commits-GitStats.R @@ -1,15 +1,33 @@ -# GitStats +commits_table_from_hosts <- purrr::list_rbind( + list( + test_mocker$use("gh_commits_table"), + test_mocker$use("gl_commits_table") + ) +) + +test_that("set_object_class works for commits", { + commits_table<- test_gitstats_priv$set_object_class( + object = commits_table_from_hosts, + class = "commits_data", + attr_list = list( + "date_range" = c("2023-06-15", "2023-06-30") + ) + ) + expect_s3_class( + commits_table, + "commits_data" + ) + expect_equal( + attr(commits_table, "date_range"), + c("2023-06-15", "2023-06-30") + ) +}) test_that("get_commits works properly", { mockery::stub( test_gitstats$get_commits, "private$get_commits_from_hosts", - purrr::list_rbind( - list( - test_mocker$use("gh_commits_table"), - test_mocker$use("gl_commits_table") - ) - ) + commits_table_from_hosts ) suppressMessages( commits_table <- test_gitstats$get_commits( From 2e277c101fa2c5a23fb82c53d8c129799fae26ff Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 17 Oct 2024 12:47:58 +0000 Subject: [PATCH 58/94] Update article. --- _pkgdown.yml | 2 +- vignettes/{get_data.Rmd => get_and_store_data.Rmd} | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) rename vignettes/{get_data.Rmd => get_and_store_data.Rmd} (94%) diff --git a/_pkgdown.yml b/_pkgdown.yml index 538aa70f..987071a6 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -17,7 +17,7 @@ articles: navbar: ~ contents: - set_hosts - - get_data + - get_and_store_data - get_repos_with_code - get_files diff --git a/vignettes/get_data.Rmd b/vignettes/get_and_store_data.Rmd similarity index 94% rename from vignettes/get_data.Rmd rename to vignettes/get_and_store_data.Rmd index a6557dfb..51dff65d 100644 --- a/vignettes/get_data.Rmd +++ b/vignettes/get_and_store_data.Rmd @@ -1,8 +1,8 @@ --- -title: "Get data" +title: "Get and store your data" output: rmarkdown::html_vignette vignette: > - %\VignetteIndexEntry{Get data} + %\VignetteIndexEntry{Get and store your data} %\VignetteEngine{knitr::rmarkdown} %\VignetteEncoding{UTF-8} --- @@ -67,7 +67,7 @@ Or globally: verbose_off(git_stats) ``` -## Storage +## Cache After pulling, the data is saved to `GitStats`. @@ -115,3 +115,11 @@ commits <- get_commits( ) dplyr::glimpse(commits) ``` + +## Storage + +Finally, have a glimpse at your storage: + +```{r} +git_stats +``` From 818e896ab4e6b3a2392d2460fd804900a2989804 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 17 Oct 2024 13:15:43 +0000 Subject: [PATCH 59/94] Improve date range attribute. The reason to set by default Sys.Date() + 1 day is to set until parameter when it is not defined. If no, no until attribute shows when printing storage which may be misleading. Also, cacheing will be flawed, as pulling commits next day without setting until will not uncache process (and it should, as data on repo may be new). Setting + 1 day ensures that whole current day will be captured. --- R/GitHost.R | 3 --- R/GitStats.R | 6 +++--- R/gitstats_functions.R | 4 ++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index 9d7015e4..c4fba300 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -133,9 +133,6 @@ GitHost <- R6::R6Class( cli::cli_alert_info("[{private$host_name}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") private$orgs <- private$engines$graphql$get_orgs() } - if (is.null(until)) { - until <- Sys.time() - } commits_table <- private$get_commits_from_orgs( since = since, until = until, diff --git a/R/GitStats.R b/R/GitStats.R index 2faccf64..a2f4d4a8 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -207,7 +207,7 @@ GitStats <- R6::R6Class( verbose = TRUE, progress = TRUE) { private$check_for_host() - args_list <- list("date_range" = c(since, until)) + args_list <- list("date_range" = c(since, as.character(until))) trigger <- private$trigger_pulling( cache = cache, storage = "commits", @@ -413,12 +413,12 @@ GitStats <- R6::R6Class( #' @param progress A logical, by default set to `verbose` value. If `FALSE` #' no `cli` progress bar will be displayed. get_release_logs = function(since, - until, + until = Sys.Date(), cache = TRUE, verbose = TRUE, progress = TRUE) { private$check_for_host() - args_list <- list("date_range" = c(since, until)) + args_list <- list("date_range" = c(since, as.character(until))) trigger <- private$trigger_pulling( storage = "release_logs", cache = cache, diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index 54c16d07..0fcef648 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -239,7 +239,7 @@ get_repos_urls <- function(gitstats_object, #' @export get_commits <- function(gitstats_object, since = NULL, - until = NULL, + until = Sys.Date() + lubridate::days(1), cache = TRUE, verbose = is_verbose(gitstats_object), progress = verbose) { @@ -501,7 +501,7 @@ get_R_package_usage <- function(gitstats_object, #' @export get_release_logs <- function(gitstats_object, since = NULL, - until = NULL, + until = Sys.Date() + lubridate::days(1), cache = TRUE, verbose = is_verbose(gitstats_object), progress = verbose) { From 703a1d7cffec12973968c6413104a5195e79f861 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 17 Oct 2024 15:00:17 +0000 Subject: [PATCH 60/94] Update function docs. --- man/get_commits.Rd | 2 +- man/get_release_logs.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/man/get_commits.Rd b/man/get_commits.Rd index 8748ea58..181817db 100644 --- a/man/get_commits.Rd +++ b/man/get_commits.Rd @@ -7,7 +7,7 @@ get_commits( gitstats_object, since = NULL, - until = NULL, + until = Sys.Date() + lubridate::days(1), cache = TRUE, verbose = is_verbose(gitstats_object), progress = verbose diff --git a/man/get_release_logs.Rd b/man/get_release_logs.Rd index bd664194..7fb365eb 100644 --- a/man/get_release_logs.Rd +++ b/man/get_release_logs.Rd @@ -7,7 +7,7 @@ get_release_logs( gitstats_object, since = NULL, - until = NULL, + until = Sys.Date() + lubridate::days(1), cache = TRUE, verbose = is_verbose(gitstats_object), progress = verbose From b69f4f7b0d89044a1edc695798f600e8e20d23a1 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 07:25:27 +0000 Subject: [PATCH 61/94] Add get_storage() function. --- NAMESPACE | 1 + NEWS.md | 1 + R/GitStats.R | 8 +++++ R/gitstats_functions.R | 18 +++++++++++ inst/WORDLIST | 2 ++ inst/get_storage_workflow.R | 32 +++++++++++++++++++ man/get_storage.Rd | 23 ++++++++++++++ tests/testthat/test-01-get_repos-GitStats.R | 1 - tests/testthat/test-get_release-GitStats.R | 21 +++++++++++-- tests/testthat/test-get_storage.R | 35 +++++++++++++++++++++ vignettes/get_and_store_data.Rmd | 15 +++++++++ 11 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 inst/get_storage_workflow.R create mode 100644 man/get_storage.Rd create mode 100644 tests/testthat/test-get_storage.R diff --git a/NAMESPACE b/NAMESPACE index e3906012..e3e5e9d1 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -10,6 +10,7 @@ export(get_files_structure) export(get_release_logs) export(get_repos) export(get_repos_urls) +export(get_storage) export(get_users) export(set_github_host) export(set_gitlab_host) diff --git a/NEWS.md b/NEWS.md index 96b57bae..bfb2c44b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,7 @@ - on the other hand, output of the function has been limited to contain only most necessary data (removing all repository stats), making thus process of obtaining package usage faster ([#474](https://github.com/r-world-devs/GitStats/issues/474)). - new `split_output` parameter has been added - when set to `TRUE` a `list` with `tibbles` (every element of the `list` for every package) instead of one `tibble` is returned. - Added possibility to get repositories for individual users with `get_repos()` ([#492](https://github.com/r-world-devs/GitStats/issues/492)). Earlier this was only possible for GitHub organizations and GitLab groups. +- Added new `get_storage()` function to retrieve data from `GitStats` object - whole or particular datasets (e.g. `commits`, `repositories` or `R_package_usage`) ([#509](https://github.com/r-world-devs/GitStats/issues/509)). ## Fixes: diff --git a/R/GitStats.R b/R/GitStats.R index a2f4d4a8..8d93cef7 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -525,6 +525,14 @@ GitStats <- R6::R6Class( private$settings$verbose }, + get_storage = function(storage) { + if (is.null(storage)) { + private$storage + } else { + private$storage[[storage]] + } + }, + #' @description A print method for a GitStats object. print = function() { cat(paste0("A ", cli::col_blue('GitStats'), " object for ", length(private$hosts), " hosts: \n")) diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index 0fcef648..f054131c 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -557,3 +557,21 @@ verbose_off <- function(gitstats_object) { is_verbose <- function(gitstats_object) { gitstats_object$is_verbose() } + +#' @title Get data from `GitStats` storage +#' @name get_storage +#' @description Retrieves whole or particular data (see `storage` parameter) +#' pulled earlier with `GitStats`. +#' @param gitstats_object A GitStats object. +#' @param storage A character, type of the data you want to get from storage: +#' `commits`, `repositories`, `release_logs`, `users`, `files`, +#' `files_structure`, `R_package_usage` or `release_logs`. +#' @return A list of tibbles (if `storage` set to `NULL`) or a tibble (if +#' `storage` defined). +#' @export +get_storage <- function(gitstats_object, + storage = NULL) { + gitstats_object$get_storage( + storage = storage + ) +} diff --git a/inst/WORDLIST b/inst/WORDLIST index d87512cf..ca88daee 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -36,3 +36,5 @@ plotly repo repos regex +tibble +tibbles diff --git a/inst/get_storage_workflow.R b/inst/get_storage_workflow.R new file mode 100644 index 00000000..10bc2221 --- /dev/null +++ b/inst/get_storage_workflow.R @@ -0,0 +1,32 @@ +devtools::load_all(".") + +git_stats <- create_gitstats() %>% + set_github_host( + orgs = c("r-world-devs", "openpharma"), + token = Sys.getenv("GITHUB_PAT") + ) + +release_logs <- get_release_logs( + gitstats_object = git_stats, + since = "2024-01-01" +) + +repos_urls <- get_repos_urls( + gitstats_object = git_stats, + with_code = "shiny" +) + +files_structure <- get_files_structure( + gitstats_object = git_stats, + pattern = "\\.md", + depth = 1L +) + +git_stats + +get_storage(git_stats) + +get_storage(git_stats, "commits") +get_storage(git_stats, "repos_urls") +get_storage(git_stats, "release_logs") +get_storage(git_stats, "files_structure") diff --git a/man/get_storage.Rd b/man/get_storage.Rd new file mode 100644 index 00000000..f73ef89a --- /dev/null +++ b/man/get_storage.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gitstats_functions.R +\name{get_storage} +\alias{get_storage} +\title{Get data from \code{GitStats} storage} +\usage{ +get_storage(gitstats_object, storage = NULL) +} +\arguments{ +\item{gitstats_object}{A GitStats object.} + +\item{storage}{A character, type of the data you want to get from storage: +\code{commits}, \code{repositories}, \code{release_logs}, \code{users}, \code{files}, +\code{files_structure}, \code{R_package_usage} or \code{release_logs}.} +} +\value{ +A list of tibbles (if \code{storage} set to \code{NULL}) or a tibble (if +\code{storage} defined). +} +\description{ +Retrieves whole or particular data (see \code{storage} parameter) +pulled earlier with \code{GitStats}. +} diff --git a/tests/testthat/test-01-get_repos-GitStats.R b/tests/testthat/test-01-get_repos-GitStats.R index 20d4f34c..6f581d96 100644 --- a/tests/testthat/test-01-get_repos-GitStats.R +++ b/tests/testthat/test-01-get_repos-GitStats.R @@ -62,7 +62,6 @@ test_that("set_object_class for repos_table works correctly", { }) test_that("get_repos works properly and for the second time uses cache", { - test_gitstats <- create_test_gitstats(hosts = 2) mockery::stub( test_gitstats$get_repos, "private$get_repos_from_hosts", diff --git a/tests/testthat/test-get_release-GitStats.R b/tests/testthat/test-get_release-GitStats.R index 19ff1dcf..c079120b 100644 --- a/tests/testthat/test-get_release-GitStats.R +++ b/tests/testthat/test-get_release-GitStats.R @@ -1,11 +1,26 @@ test_that("get_release_logs_from_hosts works as expected", { - test_gitstats <- create_test_gitstats(hosts = 2, priv_mode = TRUE) + test_gitstats_priv <- create_test_gitstats(hosts = 2, priv_mode = TRUE) mockery::stub( - test_gitstats$get_release_logs_from_hosts, + test_gitstats_priv$get_release_logs_from_hosts, "host$get_release_logs", test_mocker$use("releases_table") ) - release_logs_table <- test_gitstats$get_release_logs_from_hosts( + release_logs_table <- test_gitstats_priv$get_release_logs_from_hosts( + since = "2023-08-01", + until = "2023-09-30", + verbose = FALSE + ) + expect_releases_table(release_logs_table) + test_mocker$cache(release_logs_table) +}) + +test_that("get_release_logs works as expected", { + mockery::stub( + test_gitstats$get_release_logs, + "private$get_release_logs_from_hosts", + test_mocker$use("release_logs_table") + ) + release_logs_table <- test_gitstats$get_release_logs( since = "2023-08-01", until = "2023-09-30", verbose = FALSE diff --git a/tests/testthat/test-get_storage.R b/tests/testthat/test-get_storage.R new file mode 100644 index 00000000..d0832f7b --- /dev/null +++ b/tests/testthat/test-get_storage.R @@ -0,0 +1,35 @@ +test_that("get_storage works", { + gitstats_storage <- test_gitstats$get_storage( + storage = NULL + ) + expect_type( + gitstats_storage, + "list" + ) + expect_s3_class( + gitstats_storage[[1]], + "tbl" + ) + expect_true( + all(names(gitstats_storage) %in% c("repositories", "commits", "users", + "files", "files_structure", + "release_logs", "R_package_usage")) + ) +}) + +test_that("get_storage retrieves one table", { + gitstats_storage <- test_gitstats$get_storage( + storage = "commits" + ) + expect_s3_class( + gitstats_storage, + "tbl" + ) + expect_s3_class( + gitstats_storage, + "commits_data" + ) + expect_commits_table( + gitstats_storage + ) +}) diff --git a/vignettes/get_and_store_data.Rmd b/vignettes/get_and_store_data.Rmd index 51dff65d..92593ff0 100644 --- a/vignettes/get_and_store_data.Rmd +++ b/vignettes/get_and_store_data.Rmd @@ -123,3 +123,18 @@ Finally, have a glimpse at your storage: ```{r} git_stats ``` + +You can retrieve whole data from your `GitStats` object with: + +```{r} +get_storage(git_stats) +``` + +Or particular data set: + +```{r} +get_storage( + gitstats_object = git_stats, + storage = "repositories" +) +``` From 083501876e33736b0b878f9459553fec7297743b Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 07:42:28 +0000 Subject: [PATCH 62/94] Add test to meet coverage job. --- .github/workflows/spelling.yaml | 32 +++++++++++++++++++++++++++++++ tests/testthat/test-get_storage.R | 15 +++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 .github/workflows/spelling.yaml diff --git a/.github/workflows/spelling.yaml b/.github/workflows/spelling.yaml new file mode 100644 index 00000000..41f7b464 --- /dev/null +++ b/.github/workflows/spelling.yaml @@ -0,0 +1,32 @@ +name: 📝 Spelling Check + +on: [push, pull_request] + +jobs: + spell-check: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up R + uses: r-lib/actions/setup-r@v2 + + - name: Install R packages + run: | + install.packages('devtools') + devtools::install_github('ropensci/spelling') + + - name: Run spelling check + run: | + Rscript -e " + library(spelling); + spelling_errors <- spell_check_package(); + if (length(spelling_errors) > 0) { + print(spelling_errors); + quit(status = 1) + } else { + cat('No spelling errors found.\n') + } + " diff --git a/tests/testthat/test-get_storage.R b/tests/testthat/test-get_storage.R index d0832f7b..e350a359 100644 --- a/tests/testthat/test-get_storage.R +++ b/tests/testthat/test-get_storage.R @@ -33,3 +33,18 @@ test_that("get_storage retrieves one table", { gitstats_storage ) }) + +test_that("get_storage retrieves one table", { + gitstats_storage <- get_storage( + gitstats_object = test_gitstats, + storage = "files_structure" + ) + expect_type( + gitstats_storage, + "list" + ) + expect_s3_class( + gitstats_storage, + "files_structure" + ) +}) From d5679bc16b52f8bf9c2ac6c9ab429e68559ee0b4 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 08:57:20 +0000 Subject: [PATCH 63/94] Add example to docs. --- R/gitstats_functions.R | 15 +++++++++++++++ man/get_storage.Rd | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index f054131c..563300a0 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -568,6 +568,21 @@ is_verbose <- function(gitstats_object) { #' `files_structure`, `R_package_usage` or `release_logs`. #' @return A list of tibbles (if `storage` set to `NULL`) or a tibble (if #' `storage` defined). +#' @examples +#' \dontrun{ +#' my_gitstats <- create_gitstats() %>% +#' set_github_host( +#' token = Sys.getenv("GITHUB_PAT"), +#' orgs = c("r-world-devs", "openpharma") +#' ) +#' get_release_logs(my_gistats, since = "2024-01-01") +#' get_repos(my_gitstats) +#' +#' release_logs <- get_storage( +#' gitstats_object = my_gitstats, +#' storage = "release_logs" +#' ) +#' } #' @export get_storage <- function(gitstats_object, storage = NULL) { diff --git a/man/get_storage.Rd b/man/get_storage.Rd index f73ef89a..e2077f6e 100644 --- a/man/get_storage.Rd +++ b/man/get_storage.Rd @@ -21,3 +21,19 @@ A list of tibbles (if \code{storage} set to \code{NULL}) or a tibble (if Retrieves whole or particular data (see \code{storage} parameter) pulled earlier with \code{GitStats}. } +\examples{ +\dontrun{ + my_gitstats <- create_gitstats() \%>\% + set_github_host( + token = Sys.getenv("GITHUB_PAT"), + orgs = c("r-world-devs", "openpharma") + ) + get_release_logs(my_gistats, since = "2024-01-01") + get_repos(my_gitstats) + + release_logs <- get_storage( + gitstats_object = my_gitstats, + storage = "release_logs" + ) +} +} From f257253733ca5b5714cdfc71d79e114533051da2 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 08:57:35 +0000 Subject: [PATCH 64/94] Try fix spelling job. --- .github/workflows/spelling.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/spelling.yaml b/.github/workflows/spelling.yaml index 41f7b464..e4bc5b62 100644 --- a/.github/workflows/spelling.yaml +++ b/.github/workflows/spelling.yaml @@ -17,6 +17,7 @@ jobs: run: | install.packages('devtools') devtools::install_github('ropensci/spelling') + shell: Rscript {0} - name: Run spelling check run: | From 3b8b654f166e85cf7ad614b0cd8249349d756566 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 08:57:55 +0000 Subject: [PATCH 65/94] Prettify test job. --- .github/workflows/test-coverage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index f3419212..c8e5fef1 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -4,7 +4,7 @@ on: pull_request: branches: [devel, test, master] -name: test-coverage +name: 🧪 Test coverage jobs: test-coverage: From cc50f2fdf153f2280cd49521bf7bfe137e451116 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 09:05:58 +0000 Subject: [PATCH 66/94] Add tests to cover patch. --- tests/testthat/_snaps/z-GitStats.md | 18 ++++++++++++++++++ tests/testthat/test-get_release-GitStats.R | 1 + tests/testthat/test-z-GitStats.R | 15 +++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/tests/testthat/_snaps/z-GitStats.md b/tests/testthat/_snaps/z-GitStats.md index ffce6da0..891fca2a 100644 --- a/tests/testthat/_snaps/z-GitStats.md +++ b/tests/testthat/_snaps/z-GitStats.md @@ -49,6 +49,24 @@ ! Use either `with_code` of `with_files` parameter. i If you want to search for [shiny] code in given files - use `in_files` parameter together with `with_code` instead. +# print_storage_attribute + + Code + test_gitstats_priv$print_storage_attribute(storage_data = test_mocker$use( + "commits_table"), storage_name = "commits") + Output + + [1] [date range: 2023-06-15 - 2023-06-30] + +--- + + Code + test_gitstats_priv$print_storage_attribute(storage_data = test_mocker$use( + "release_logs_table"), storage_name = "release_logs") + Output + + [1] [date range: 2023-08-01 - 2023-09-30] + # subgroups are cleanly printed in GitStats Code diff --git a/tests/testthat/test-get_release-GitStats.R b/tests/testthat/test-get_release-GitStats.R index c079120b..20cccafe 100644 --- a/tests/testthat/test-get_release-GitStats.R +++ b/tests/testthat/test-get_release-GitStats.R @@ -26,4 +26,5 @@ test_that("get_release_logs works as expected", { verbose = FALSE ) expect_releases_table(release_logs_table) + test_mocker$cache(release_logs_table) }) diff --git a/tests/testthat/test-z-GitStats.R b/tests/testthat/test-z-GitStats.R index 1782bbe5..31df2b45 100644 --- a/tests/testthat/test-z-GitStats.R +++ b/tests/testthat/test-z-GitStats.R @@ -84,6 +84,21 @@ test_that("check_if_args_changed", { ) }) +test_that("print_storage_attribute", { + expect_snapshot( + test_gitstats_priv$print_storage_attribute( + storage_data = test_mocker$use("commits_table"), + storage_name = "commits" + ) + ) + expect_snapshot( + test_gitstats_priv$print_storage_attribute( + storage_data = test_mocker$use("release_logs_table"), + storage_name = "release_logs" + ) + ) +}) + test_that("show_orgs print orgs properly", { test_gitstats <- create_test_gitstats(hosts = 2) expect_equal( From f107d3c4b14d1bf16462e07547f16cc217c87b48 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 09:26:20 +0000 Subject: [PATCH 67/94] Bump version and add a job. --- .github/workflows/version.yaml | 45 ++++++++++++++++++++++++++++++++++ DESCRIPTION | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/version.yaml diff --git a/.github/workflows/version.yaml b/.github/workflows/version.yaml new file mode 100644 index 00000000..ecae3763 --- /dev/null +++ b/.github/workflows/version.yaml @@ -0,0 +1,45 @@ +name: ⬆️ Check Version Bump + +on: + pull_request: + paths: + - DESCRIPTION + +jobs: + check-version: + runs-on: ubuntu-latest + + steps: + - name: Checkout source branch + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: source + + - name: Checkout target branch + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: target + + - name: Compare versions + id: compare_versions + run: | + SOURCE_VERSION=$(awk '/^Version:/ { print $2 }' source/DESCRIPTION) + TARGET_VERSION=$(awk '/^Version:/ { print $2 }' target/DESCRIPTION) + echo "Source package version: $SOURCE_VERSION" + echo "Target package version: $TARGET_VERSION" + + if [ "$SOURCE_VERSION" == "$TARGET_VERSION" ]; then + echo "Versions are identical" + echo "::set-output name=versions_identical::true" + else + echo "Versions differ" + echo "::set-output name=versions_identical::false" + fi + + - name: Fail if versions are identical + if: steps.compare_versions.outputs.versions_identical == 'true' + run: | + echo "The package versions are identical between the source and target branches." + exit 1 diff --git a/DESCRIPTION b/DESCRIPTION index 99e7d83e..e284f541 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.0.9005 +Version: 2.1.0.9006 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), From 14c34ed96a893bc756fe22e0857c6d2bf0737c92 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 09:26:42 +0000 Subject: [PATCH 68/94] Try fix spelling job. --- .github/workflows/spelling.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/spelling.yaml b/.github/workflows/spelling.yaml index e4bc5b62..b49c1e74 100644 --- a/.github/workflows/spelling.yaml +++ b/.github/workflows/spelling.yaml @@ -13,6 +13,11 @@ jobs: - name: Set up R uses: r-lib/actions/setup-r@v2 + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libcurl4-openssl-dev + - name: Install R packages run: | install.packages('devtools') From 0f5b2fdce798ccf1866e09d10d53429e0f2d007c Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 09:26:57 +0000 Subject: [PATCH 69/94] Add icons. --- .github/workflows/R-CMD-check.yaml | 2 +- .github/workflows/lint.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 76d28d4f..bf4a361c 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -12,7 +12,7 @@ jobs: R-CMD-check: runs-on: ${{ matrix.config.os }} - name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + name: 👮 ${{ matrix.config.os }} (${{ matrix.config.r }}) strategy: fail-fast: false diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index e252d22b..280eef9e 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -2,7 +2,7 @@ # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help on: [push, pull_request] -name: lint +name: 🕵 Lint jobs: lint: From e21916bac936ec3679babc4a9f1b4566c4e6e8e4 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 09:38:49 +0000 Subject: [PATCH 70/94] Try fix spelling job, and some changes to jobs. --- .github/workflows/R-CMD-check.yaml | 4 ++-- .github/workflows/lint.yaml | 10 ++++++---- .github/workflows/spelling.yaml | 7 ++++++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index bf4a361c..58c48806 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -6,13 +6,13 @@ on: pull_request: branches: [devel, test, master] -name: R-CMD-check +name: 👮 R-CMD-check jobs: R-CMD-check: runs-on: ${{ matrix.config.os }} - name: 👮 ${{ matrix.config.os }} (${{ matrix.config.r }}) + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) strategy: fail-fast: false diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 280eef9e..a96edc01 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,9 +1,11 @@ -# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples -# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help -on: [push, pull_request] - name: 🕵 Lint +on: + push: + branches: [master] + pull_request: + branches: [devel, test, master] + jobs: lint: runs-on: ubuntu-latest diff --git a/.github/workflows/spelling.yaml b/.github/workflows/spelling.yaml index b49c1e74..3071600d 100644 --- a/.github/workflows/spelling.yaml +++ b/.github/workflows/spelling.yaml @@ -1,6 +1,10 @@ name: 📝 Spelling Check -on: [push, pull_request] +on: + push: + branches: [master] + pull_request: + branches: [devel, test, master] jobs: spell-check: @@ -17,6 +21,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libcurl4-openssl-dev + sudo apt-get install -y libharfbuzz-dev libfribidi-dev - name: Install R packages run: | From 9e63c47f96dbd88f27a8f6e775af9767f8e41b18 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 09:54:04 +0000 Subject: [PATCH 71/94] Try fix spelling job. --- .github/workflows/spelling.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/spelling.yaml b/.github/workflows/spelling.yaml index 3071600d..2e404a5e 100644 --- a/.github/workflows/spelling.yaml +++ b/.github/workflows/spelling.yaml @@ -1,4 +1,4 @@ -name: 📝 Spelling Check +name: 📝 Spelling on: push: @@ -14,19 +14,16 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 + - name: Set up R dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + - name: Set up R uses: r-lib/actions/setup-r@v2 - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libcurl4-openssl-dev - sudo apt-get install -y libharfbuzz-dev libfribidi-dev - - name: Install R packages run: | install.packages('devtools') - devtools::install_github('ropensci/spelling') + install.packages('spelling') shell: Rscript {0} - name: Run spelling check From 35c0ca1c31b212afc9bd3f16d435b868ea995ef4 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 10:00:01 +0000 Subject: [PATCH 72/94] Try last fix. --- .github/workflows/spelling.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/spelling.yaml b/.github/workflows/spelling.yaml index 2e404a5e..4faf9099 100644 --- a/.github/workflows/spelling.yaml +++ b/.github/workflows/spelling.yaml @@ -14,12 +14,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up R dependencies - uses: r-lib/actions/setup-r-dependencies@v2 - - name: Set up R uses: r-lib/actions/setup-r@v2 + - name: Set up R dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + - name: Install R packages run: | install.packages('devtools') From 6893eae951bb78dc5c019a5617d9a74bf9a83745 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 10:07:05 +0000 Subject: [PATCH 73/94] Remove. --- .github/workflows/spelling.yaml | 40 --------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/spelling.yaml diff --git a/.github/workflows/spelling.yaml b/.github/workflows/spelling.yaml deleted file mode 100644 index 4faf9099..00000000 --- a/.github/workflows/spelling.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: 📝 Spelling - -on: - push: - branches: [master] - pull_request: - branches: [devel, test, master] - -jobs: - spell-check: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up R - uses: r-lib/actions/setup-r@v2 - - - name: Set up R dependencies - uses: r-lib/actions/setup-r-dependencies@v2 - - - name: Install R packages - run: | - install.packages('devtools') - install.packages('spelling') - shell: Rscript {0} - - - name: Run spelling check - run: | - Rscript -e " - library(spelling); - spelling_errors <- spell_check_package(); - if (length(spelling_errors) > 0) { - print(spelling_errors); - quit(status = 1) - } else { - cat('No spelling errors found.\n') - } - " From d8e73b915c50d3eec7b9df10398f7dae43453ab3 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 10:54:05 +0000 Subject: [PATCH 74/94] Add possibility to pass host with `http`. --- DESCRIPTION | 2 +- NEWS.md | 1 + R/GitHost.R | 5 ++++- R/GitHostGitHub.R | 1 + tests/testthat/test-GitHost-helpers.R | 20 +++++++++++++++++++- 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index e284f541..116fcf87 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.0.9006 +Version: 2.1.0.9007 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), diff --git a/NEWS.md b/NEWS.md index bfb2c44b..1392b25b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,6 +14,7 @@ - Fixed getting large search responses for GitHub ([#491](https://github.com/r-world-devs/GitStats/issues/491)). - Fixed checking token scopes ([#501](https://github.com/r-world-devs/GitStats/issues/501)). If token scopes are insufficient error is returned and `GitHost` is not passed to `GitStats`. This also applies to situation when `GitStats` looks for default tokens (not defined by user). Earlier, if tests for token failed, an empty token was passed and `GitStats` was created, which was misleading for the user. - User can now optionally pass public GitHub host name (`github.com` or `https://github.com`) to `set_github_host()` ([#475](https://github.com/r-world-devs/GitStats/issues/475)). +- It is possible to pass hosts in more flexible way than before (e.g. `{host_url}`, `http://{host_url}` or `https://{host_url}`) to `host` parameter in `set_*_host() function ([#399](https://github.com/r-world-devs/GitStats/issues/399)). # GitStats 2.1.0 diff --git a/R/GitHost.R b/R/GitHost.R index c4fba300..3e325083 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -297,11 +297,14 @@ GitHost <- R6::R6Class( # Set API url set_custom_api_url = function(host) { - private$api_url <- if (!grepl("https://", host)) { + private$api_url <- if (!grepl("https|http", host)) { glue::glue( "https://{host}/api/v{private$api_version}" ) } else { + if (grepl("http(?!s)", host, perl = TRUE)) { + host <- gsub("http", "https", host) + } glue::glue( "{host}/api/v{private$api_version}" ) diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 24cdc67f..f4e570df 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -56,6 +56,7 @@ GitHostGitHub <- R6::R6Class( set_api_url = function(host) { if (is.null(host) || host == "https://github.com" || + host == "http://github.com" || host == "github.com") { private$api_url <- "https://api.github.com" } else { diff --git a/tests/testthat/test-GitHost-helpers.R b/tests/testthat/test-GitHost-helpers.R index d42e58fe..fbadb260 100644 --- a/tests/testthat/test-GitHost-helpers.R +++ b/tests/testthat/test-GitHost-helpers.R @@ -22,7 +22,7 @@ test_that("set_owner_types sets attributes to owners list", { expect_equal(owner[[1]], "test_org", ignore_attr = TRUE) }) -test_that("set_api_url works for public hosts", { +test_that("set_api_url works", { expect_equal({ github_testhost_priv$set_api_url( host = "github.com" @@ -33,6 +33,11 @@ test_that("set_api_url works for public hosts", { host = "https://github.com" ) }, "https://api.github.com") + expect_equal({ + github_testhost_priv$set_api_url( + host = "http://github.com" + ) + }, "https://api.github.com") expect_equal({ github_testhost_priv$set_api_url( host = "https://github.company.com" @@ -49,3 +54,16 @@ test_that("set_api_url works for public hosts", { ) }, "https://gitlab.com/api/v4") }) + +test_that("set_custom_api_url works", { + expect_equal({ + gitlab_testhost_priv$set_custom_api_url( + host = "http://gitlab.com" + ) + }, "https://gitlab.com/api/v4") + expect_equal({ + github_testhost_priv$set_custom_api_url( + host = "http://github.company.com" + ) + }, "https://github.company.com/api/v3") +}) From fbd65a254adfa3dc8d58479d64bdf2d9d28ea6a5 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 11:11:32 +0000 Subject: [PATCH 75/94] Bump version, update NEWS. --- DESCRIPTION | 2 +- NEWS.md | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 116fcf87..5cf3f4cc 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.0.9007 +Version: 2.1.1 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), diff --git a/NEWS.md b/NEWS.md index 1392b25b..8d5645fc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,11 @@ -# GitStats (development version) +# GitStats 2.1.1 + +This is a patch release which introduces some improvements in `get_R_package_usage()` on speed and possibility to pull at once data on multiple R packages, new `get_storage()` function and some fixes for checking token scopes and setting hosts. ## Features: - Optimized `get_R_package_usage()` function: - - you can now pass a vector of packages names (new `packages` parameter replacing old `package_name`) ([#494](https://github.com/r-world-devs/GitStats/issues/494)), + - it is now possible to pass a vector of packages names (new `packages` parameter replacing old `package_name`) ([#494](https://github.com/r-world-devs/GitStats/issues/494)), - on the other hand, output of the function has been limited to contain only most necessary data (removing all repository stats), making thus process of obtaining package usage faster ([#474](https://github.com/r-world-devs/GitStats/issues/474)). - new `split_output` parameter has been added - when set to `TRUE` a `list` with `tibbles` (every element of the `list` for every package) instead of one `tibble` is returned. - Added possibility to get repositories for individual users with `get_repos()` ([#492](https://github.com/r-world-devs/GitStats/issues/492)). Earlier this was only possible for GitHub organizations and GitLab groups. @@ -13,11 +15,13 @@ - Fixed getting large search responses for GitHub ([#491](https://github.com/r-world-devs/GitStats/issues/491)). - Fixed checking token scopes ([#501](https://github.com/r-world-devs/GitStats/issues/501)). If token scopes are insufficient error is returned and `GitHost` is not passed to `GitStats`. This also applies to situation when `GitStats` looks for default tokens (not defined by user). Earlier, if tests for token failed, an empty token was passed and `GitStats` was created, which was misleading for the user. -- User can now optionally pass public GitHub host name (`github.com` or `https://github.com`) to `set_github_host()` ([#475](https://github.com/r-world-devs/GitStats/issues/475)). -- It is possible to pass hosts in more flexible way than before (e.g. `{host_url}`, `http://{host_url}` or `https://{host_url}`) to `host` parameter in `set_*_host() function ([#399](https://github.com/r-world-devs/GitStats/issues/399)). +- It is now possible to pass public GitHub host name (`github.com` or `https://github.com`) to `set_github_host()` ([#475](https://github.com/r-world-devs/GitStats/issues/475)). +- It is also possible to pass hosts in more flexible way than before (e.g. `{host_url}`, `http://{host_url}` or `https://{host_url}`) to `host` parameter in `set_*_host() function ([#399](https://github.com/r-world-devs/GitStats/issues/399)). # GitStats 2.1.0 +This minor release comes up with new `get_files_structure()` function and adjustments to `get_files_content()` so user can pull custom (by defining pattern of files and depth of directories) files tree from repository and pull their content. + ## New features: - Added new `get_files_structure()` function to pull files structure for a given repository with possibility to control level of directories (`depth` parameter) and to limit output to files matching regex argument passed to `pattern` parameter ([#338](https://github.com/r-world-devs/GitStats/issues/338)). Together with that, `get_files()` function was renamed to `get_files_content()` to better reflect its purpose. From 93c6adff593a2d7be38f0b9f23798c1b079c73d0 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 11:13:47 +0000 Subject: [PATCH 76/94] Prettify build site job. --- .github/workflows/pkgdown.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 7d1ee754..fe54c7b4 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -9,7 +9,7 @@ on: types: [published] workflow_dispatch: -name: pkgdown.yaml +name: 🌐 Build site permissions: read-all From f3ee13fe9db09912e2cabee0b4ab17d385f534c9 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 12:03:12 +0000 Subject: [PATCH 77/94] Adjust package to submit to CRAN. --- .Rbuildignore | 4 +++ README.Rmd | 6 +++++ README.md | 6 +++++ cran-comments.md | 5 ++++ tests/testthat/setup.R | 50 ----------------------------------- tests/testthat/test-helpers.R | 1 + vignettes/set_hosts.Rmd | 4 +-- 7 files changed, 24 insertions(+), 52 deletions(-) create mode 100644 cran-comments.md diff --git a/.Rbuildignore b/.Rbuildignore index d31234c8..c09fe20f 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -13,3 +13,7 @@ ^pkgdown$ ^project_metadata.yaml ^.lintr +^CRAN-SUBMISSION$ +^cran-comments\.md$ +^vignettes/get_and_store_data\.Rmd$ +^vignettes/get_files\.Rmd$ diff --git a/README.Rmd b/README.Rmd index d8319f04..8f6de2aa 100644 --- a/README.Rmd +++ b/README.Rmd @@ -34,6 +34,12 @@ With GitStats you can pull git data in a uniform way (table format) from GitHub ## Installation ```r +From CRAN: + +install.packages("GitStats") + +Or development version: + devtools::install_github("r-world-devs/GitStats") ``` diff --git a/README.md b/README.md index f0b46433..decb72a6 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,12 @@ GitHub and GitLab. For the time-being you can get data on: ## Installation ``` r +From CRAN: + +install.packages("GitStats") + +Or development version: + devtools::install_github("r-world-devs/GitStats") ``` diff --git a/cran-comments.md b/cran-comments.md new file mode 100644 index 00000000..858617db --- /dev/null +++ b/cran-comments.md @@ -0,0 +1,5 @@ +## R CMD check results + +0 errors | 0 warnings | 1 note + +* This is a new release. diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 0b13ecd4..9704d42a 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -1,53 +1,3 @@ -if (nchar(Sys.getenv("GITHUB_PAT")) == 0) { - cli::cli_abort(c( - "x" = "You did not set up your GITHUB_PAT environment variable.", - "i" = "If you wish to run tests for GitHub - set up your GITHUB_PAT - enviroment variable (as a GitHub access token on github.com)." - )) -} -if (nchar(Sys.getenv("GITHUB_PAT")) > 0) { - tryCatch({ - httr2::request("https://api.github.com") %>% - httr2::req_headers("Authorization" = paste0("Bearer ", Sys.getenv("GITHUB_PAT"))) %>% - httr2::req_perform() - }, - error = function(e) { - if (grepl("401", e$message)) { - cli::cli_abort(c( - "x" = "Your GITHUB_PAT enviroment variable does not grant access. Please - setup your GITHUB_PAT before running tests.", - "i" = "If you wish to run tests for GitHub - set up your GITHUB_PAT - enviroment variable (as a GitHub access token on github.com)." - )) - } - }) -} -if (nchar(Sys.getenv("GITLAB_PAT_PUBLIC")) == 0) { - cli::cli_abort(c( - "x" = "You did not set up your GITLAB_PAT_PUBLIC environment variable.", - "i" = "If you wish to run tests for GitLab - set up your GITLAB_PAT_PUBLIC - enviroment variable (as a GitLab access token on gitlab.com)." - )) -} -if (nchar(Sys.getenv("GITLAB_PAT_PUBLIC")) > 0) { - tryCatch({ - httr2::request("https://gitlab.com/api/v4/projects") %>% - httr2::req_headers("Authorization" = paste0("Bearer ", - Sys.getenv("GITLAB_PAT_PUBLIC"))) %>% - httr2::req_perform() - }, - error = function(e) { - if (grepl("401", e$message)) { - cli::cli_abort(c( - "x" = "Your GITLAB_PAT_PUBLIC enviroment variable does not grant access. - Please setup your GITLAB_PAT_PUBLIC before running tests.", - "i" = "If you wish to run tests for GitLab - set up your GITLAB_PAT_PUBLIC - enviroment variable (as a GitLab access token on gitlab.com)." - )) - } - }) -} - test_mocker <- Mocker$new() test_gitstats <- create_test_gitstats(hosts = 2) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 34c498bb..86b4a349 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -152,6 +152,7 @@ test_that("`test_token` works properly", { }) test_that("`set_default_token` sets default token for GitLab", { + skip_on_cran() expect_snapshot( withr::with_envvar(new = c("GITLAB_PAT" = Sys.getenv("GITLAB_PAT_PUBLIC")), { default_token <- gitlab_testhost_priv$set_default_token(verbose = TRUE) diff --git a/vignettes/set_hosts.Rmd b/vignettes/set_hosts.Rmd index 976b7dc4..74734a71 100644 --- a/vignettes/set_hosts.Rmd +++ b/vignettes/set_hosts.Rmd @@ -33,7 +33,7 @@ When setting hosts you need to take into account: If you connect to public hosts you simply call `set_github_host()` or `set_gitlab_host()` function without specifying `host` parameter. -```{r} +```{r, eval = FALSE} library(GitStats) git_stats <- create_gitstats() %>% set_github_host( @@ -72,7 +72,7 @@ When setting hosts you choose what scanning scope of your GitStats will be: * `organizations/groups` - in this case you need to pass character arguments (names of organizations (in case of GitHub) or groups (in case of GitLab)) to `orgs` parameter. -```{r} +```{r, eval = FALSE} git_stats <- create_gitstats() %>% set_github_host( orgs = c("r-world-devs", "openpharma"), From 01a48429bf8496bcd510472b999dff8d6bc6d955 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 13:24:50 +0000 Subject: [PATCH 78/94] Remove snapshots so tests can pass on CRAN. Tests with snapshots are skipped on CRAN which results in case of GitStats for some tests not passing outputs to test_mocker and in result failutre of following up tests. --- tests/testthat/_snaps/01-get_repos-GitHub.md | 15 -------------- .../testthat/_snaps/01-get_repos-GitStats.md | 8 -------- tests/testthat/_snaps/get_commits-GitLab.md | 8 -------- .../_snaps/get_files_content-GitLab.md | 10 ---------- .../_snaps/get_files_structure-GitLab.md | 9 --------- .../_snaps/get_files_structure-GitStats.md | 6 ------ tests/testthat/test-01-get_repos-GitHub.R | 11 +++++----- tests/testthat/test-01-get_repos-GitStats.R | 4 ++-- tests/testthat/test-get_commits-GitLab.R | 8 +++----- .../testthat/test-get_files_content-GitLab.R | 20 +++++++++---------- .../test-get_files_structure-GitLab.R | 12 +++++------ .../test-get_files_structure-GitStats.R | 10 ++++------ 12 files changed, 28 insertions(+), 93 deletions(-) delete mode 100644 tests/testthat/_snaps/01-get_repos-GitStats.md delete mode 100644 tests/testthat/_snaps/get_commits-GitLab.md diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index 9ca42bbd..0e96fbe5 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -5,14 +5,6 @@ Output [1] "\n query GetReposByOrg($login: String! $repoCursor: String!) {\n repositoryOwner(login: $login) {\n ... on Organization {\n \n repositories(first: 100 after: $repoCursor) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n \n }\n }\n }" -# `prepare_repos_table()` prepares repos table - - Code - gh_repos_by_code_table <- test_rest_github$prepare_repos_table(repos_list = test_mocker$ - use("gh_repos_by_code_tailored")) - Message - i Preparing repositories table... - # `prepare_repos_table()` prepares minimum version of repos table Code @@ -21,13 +13,6 @@ Message i Preparing repositories table... -# `get_all_repos()` works as expected - - Code - gh_repos_table <- github_testhost_priv$get_all_repos() - Message - i [Host:GitHub][Engine:GraphQl][Scope:test-org] Pulling repositories... - # `get_repos_contributors()` works on GitHost level Code diff --git a/tests/testthat/_snaps/01-get_repos-GitStats.md b/tests/testthat/_snaps/01-get_repos-GitStats.md deleted file mode 100644 index a9bca13f..00000000 --- a/tests/testthat/_snaps/01-get_repos-GitStats.md +++ /dev/null @@ -1,8 +0,0 @@ -# get_repos works properly and for the second time uses cache - - Code - repos_table <- test_gitstats$get_repos() - Message - ! Retrieving repositories from the GitStats storage. - i If you wish to pull the data from API once more, set `cache` parameter to `FALSE`. - diff --git a/tests/testthat/_snaps/get_commits-GitLab.md b/tests/testthat/_snaps/get_commits-GitLab.md deleted file mode 100644 index 439f03b4..00000000 --- a/tests/testthat/_snaps/get_commits-GitLab.md +++ /dev/null @@ -1,8 +0,0 @@ -# `get_commits_authors_handles_and_names()` adds author logis and names to commits table - - Code - gl_commits_table <- test_rest_gitlab$get_commits_authors_handles_and_names( - commits_table = test_mocker$use("gl_commits_table"), verbose = TRUE) - Message - i Looking up for authors' names and logins... - diff --git a/tests/testthat/_snaps/get_files_content-GitLab.md b/tests/testthat/_snaps/get_files_content-GitLab.md index 076446d5..f7b8d5dd 100644 --- a/tests/testthat/_snaps/get_files_content-GitLab.md +++ b/tests/testthat/_snaps/get_files_content-GitLab.md @@ -12,13 +12,3 @@ Output [1] "\n query GetFilesByRepo($fullPath: ID!, $file_paths: [String!]!) {\n project(fullPath: $fullPath) {\n name\n id\n webUrl\n repository {\n blobs(paths: $file_paths) {\n nodes {\n name\n rawBlob\n size\n }\n }\n }\n }\n }\n " -# Gitlab GraphQL switches to pulling files per repositories when query is too complex - - Code - gitlab_files_response_by_repos <- test_graphql_gitlab$get_files_from_org(org = "mbtests", - type = "organization", repos = NULL, file_paths = c("DESCRIPTION", - "project_metadata.yaml", "README.md"), host_files_structure = NULL, - only_text_files = TRUE, verbose = TRUE, progress = FALSE) - Message - i I will switch to pulling files per repository. - diff --git a/tests/testthat/_snaps/get_files_structure-GitLab.md b/tests/testthat/_snaps/get_files_structure-GitLab.md index 5fdb0493..342d0ead 100644 --- a/tests/testthat/_snaps/get_files_structure-GitLab.md +++ b/tests/testthat/_snaps/get_files_structure-GitLab.md @@ -5,15 +5,6 @@ Output [1] "\n query GetFilesTree ($fullPath: ID!, $file_path: String!) {\n project(fullPath: $fullPath) {\n repository {\n tree(path: $file_path) {\n trees (first: 100) {\n pageInfo{\n endCursor\n hasNextPage\n }\n nodes {\n name\n }\n }\n blobs (first: 100) {\n pageInfo{\n endCursor\n hasNextPage\n }\n nodes {\n name\n }\n }\n }\n }\n }\n }\n " -# get_files_structure_from_orgs pulls files structure for repositories in orgs - - Code - gl_files_structure_from_orgs <- gitlab_testhost_priv$ - get_files_structure_from_orgs(pattern = "\\.md", depth = 1L, verbose = TRUE, - progress = FALSE) - Message - i [Host:GitLab][Engine:GraphQl][Scope:mbtests] Pulling files structure...[files matching pattern: '\.md']... - # get_files_content makes use of files_structure Code diff --git a/tests/testthat/_snaps/get_files_structure-GitStats.md b/tests/testthat/_snaps/get_files_structure-GitStats.md index 1f0e3190..a3cf4497 100644 --- a/tests/testthat/_snaps/get_files_structure-GitStats.md +++ b/tests/testthat/_snaps/get_files_structure-GitStats.md @@ -7,9 +7,3 @@ ! No files structure found for matching pattern \.png in 1 level of dirs. ! Files structure will not be saved in GitStats. -# get_files_structure works as expected - - Code - files_structure <- test_gitstats$get_files_structure(pattern = "\\.md", depth = 2L, - verbose = TRUE) - diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 6111ee36..02920973 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -226,10 +226,9 @@ test_that("GitHub tailors `repos_list` to minimal version of table", { }) test_that("`prepare_repos_table()` prepares repos table", { - expect_snapshot( - gh_repos_by_code_table <- test_rest_github$prepare_repos_table( - repos_list = test_mocker$use("gh_repos_by_code_tailored") - ) + gh_repos_by_code_table <- test_rest_github$prepare_repos_table( + repos_list = test_mocker$use("gh_repos_by_code_tailored"), + verbose = FALSE ) expect_repos_table( gh_repos_by_code_table @@ -399,8 +398,8 @@ test_that("`get_all_repos()` works as expected", { "graphql_engine$prepare_repos_table", test_mocker$use("gh_repos_table") ) - expect_snapshot( - gh_repos_table <- github_testhost_priv$get_all_repos() + gh_repos_table <- github_testhost_priv$get_all_repos( + verbose = FALSE ) expect_repos_table( gh_repos_table diff --git a/tests/testthat/test-01-get_repos-GitStats.R b/tests/testthat/test-01-get_repos-GitStats.R index 6f581d96..90bcdd6f 100644 --- a/tests/testthat/test-01-get_repos-GitStats.R +++ b/tests/testthat/test-01-get_repos-GitStats.R @@ -72,8 +72,8 @@ test_that("get_repos works properly and for the second time uses cache", { repos_object = repos_table, with_cols = c("contributors", "contributors_n") ) - expect_snapshot( - repos_table <- test_gitstats$get_repos() + repos_table <- test_gitstats$get_repos( + verbose = FALSE ) expect_repos_table_object( repos_object = repos_table, diff --git a/tests/testthat/test-get_commits-GitLab.R b/tests/testthat/test-get_commits-GitLab.R index d75360e0..bfca6d1d 100644 --- a/tests/testthat/test-get_commits-GitLab.R +++ b/tests/testthat/test-get_commits-GitLab.R @@ -61,11 +61,9 @@ test_that("`get_commits_authors_handles_and_names()` adds author logis and names "private$get_authors_dict", test_mocker$use("authors_dict") ) - expect_snapshot( - gl_commits_table <- test_rest_gitlab$get_commits_authors_handles_and_names( - commits_table = test_mocker$use("gl_commits_table"), - verbose = TRUE - ) + gl_commits_table <- test_rest_gitlab$get_commits_authors_handles_and_names( + commits_table = test_mocker$use("gl_commits_table"), + verbose = FALSE ) expect_commits_table( gl_commits_table, diff --git a/tests/testthat/test-get_files_content-GitLab.R b/tests/testthat/test-get_files_content-GitLab.R index 45e40f38..1c84606a 100644 --- a/tests/testthat/test-get_files_content-GitLab.R +++ b/tests/testthat/test-get_files_content-GitLab.R @@ -111,17 +111,15 @@ test_that("Gitlab GraphQL switches to pulling files per repositories when query "private$is_complexity_error", TRUE ) - expect_snapshot( - gitlab_files_response_by_repos <- test_graphql_gitlab$get_files_from_org( - org = "mbtests", - type = "organization", - repos = NULL, - file_paths = c("DESCRIPTION", "project_metadata.yaml", "README.md"), - host_files_structure = NULL, - only_text_files = TRUE, - verbose = TRUE, - progress = FALSE - ) + gitlab_files_response_by_repos <- test_graphql_gitlab$get_files_from_org( + org = "mbtests", + type = "organization", + repos = NULL, + file_paths = c("DESCRIPTION", "project_metadata.yaml", "README.md"), + host_files_structure = NULL, + only_text_files = TRUE, + verbose = FALSE, + progress = FALSE ) expect_gitlab_files_from_org_by_repos_response( response = gitlab_files_response_by_repos, diff --git a/tests/testthat/test-get_files_structure-GitLab.R b/tests/testthat/test-get_files_structure-GitLab.R index 5898ec98..e2bf6450 100644 --- a/tests/testthat/test-get_files_structure-GitLab.R +++ b/tests/testthat/test-get_files_structure-GitLab.R @@ -174,13 +174,11 @@ test_that("get_files_structure_from_orgs pulls files structure for repositories "graphql_engine$get_files_structure_from_org", test_mocker$use("gl_files_structure_shallow") ) - expect_snapshot( - gl_files_structure_from_orgs <- gitlab_testhost_priv$get_files_structure_from_orgs( - pattern = "\\.md", - depth = 1L, - verbose = TRUE, - progress = FALSE - ) + gl_files_structure_from_orgs <- gitlab_testhost_priv$get_files_structure_from_orgs( + pattern = "\\.md", + depth = 1L, + verbose = FALSE, + progress = FALSE ) expect_equal( names(gl_files_structure_from_orgs), diff --git a/tests/testthat/test-get_files_structure-GitStats.R b/tests/testthat/test-get_files_structure-GitStats.R index eff6fb16..280f5873 100644 --- a/tests/testthat/test-get_files_structure-GitStats.R +++ b/tests/testthat/test-get_files_structure-GitStats.R @@ -37,12 +37,10 @@ test_that("get_files_structure works as expected", { "private$get_files_structure_from_hosts", test_mocker$use("files_structure_from_hosts") ) - expect_snapshot( - files_structure <- test_gitstats$get_files_structure( - pattern = "\\.md", - depth = 2L, - verbose = TRUE - ) + files_structure <- test_gitstats$get_files_structure( + pattern = "\\.md", + depth = 2L, + verbose = FALSE ) expect_s3_class(files_structure, "files_structure") test_mocker$cache(files_structure) From ae3f066a5518dd98cb37c183ad6cd7211d8209aa Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 13:44:42 +0000 Subject: [PATCH 79/94] Follow up and try to fix coverage patch job. --- tests/testthat/_snaps/01-get_repos-GitHub.md | 17 ----------------- tests/testthat/_snaps/01-get_repos-GitLab.md | 8 -------- tests/testthat/_snaps/get_commits-GitStats.md | 5 +++++ tests/testthat/test-01-get_repos-GitHub.R | 19 ++++++++----------- tests/testthat/test-01-get_repos-GitLab.R | 7 +++---- tests/testthat/test-get_commits-GitStats.R | 7 +++++++ 6 files changed, 23 insertions(+), 40 deletions(-) create mode 100644 tests/testthat/_snaps/get_commits-GitStats.md diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index 0e96fbe5..b8230cb7 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -5,20 +5,3 @@ Output [1] "\n query GetReposByOrg($login: String! $repoCursor: String!) {\n repositoryOwner(login: $login) {\n ... on Organization {\n \n repositories(first: 100 after: $repoCursor) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n \n }\n }\n }" -# `prepare_repos_table()` prepares minimum version of repos table - - Code - gh_repos_by_code_table_min <- test_rest_github$prepare_repos_table(repos_list = test_mocker$ - use("gh_repos_by_code_tailored_min"), output = "table_min") - Message - i Preparing repositories table... - -# `get_repos_contributors()` works on GitHost level - - Code - gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( - repos_table = test_mocker$use("gh_repos_table_with_platform"), verbose = TRUE, - progress = FALSE) - Message - i [Host:GitHub][Engine:REST] Pulling contributors... - diff --git a/tests/testthat/_snaps/01-get_repos-GitLab.md b/tests/testthat/_snaps/01-get_repos-GitLab.md index b1db1759..632c8019 100644 --- a/tests/testthat/_snaps/01-get_repos-GitLab.md +++ b/tests/testthat/_snaps/01-get_repos-GitLab.md @@ -5,11 +5,3 @@ Output [1] "\n query GetReposByOrg($org: ID! $repo_cursor: String!) {\n group(fullPath: $org) {\n projects(first: 100 after: $repo_cursor) {\n \n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n namespace {\n path\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" -# REST client prepares table from GitLab repositories response - - Code - gl_repos_by_code_table <- test_rest_gitlab$prepare_repos_table(repos_list = test_mocker$ - use("gl_repos_by_code_tailored")) - Message - i Preparing repositories table... - diff --git a/tests/testthat/_snaps/get_commits-GitStats.md b/tests/testthat/_snaps/get_commits-GitStats.md new file mode 100644 index 00000000..9eace1d7 --- /dev/null +++ b/tests/testthat/_snaps/get_commits-GitStats.md @@ -0,0 +1,5 @@ +# get_commits_stats returns error when no commits + + x No commits found in GitStats storage. + i Run first `get_commits()`. + diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 02920973..cf6eb277 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -237,11 +237,10 @@ test_that("`prepare_repos_table()` prepares repos table", { }) test_that("`prepare_repos_table()` prepares minimum version of repos table", { - expect_snapshot( - gh_repos_by_code_table_min <- test_rest_github$prepare_repos_table( - repos_list = test_mocker$use("gh_repos_by_code_tailored_min"), - output = "table_min" - ) + gh_repos_by_code_table_min <- test_rest_github$prepare_repos_table( + repos_list = test_mocker$use("gh_repos_by_code_tailored_min"), + output = "table_min", + verbose = FALSE ) expect_repos_table( gh_repos_by_code_table_min, @@ -468,12 +467,10 @@ test_that("`get_repos_contributors()` works on GitHost level", { "rest_engine$get_repos_contributors", test_mocker$use("gh_repos_with_contributors") ) - expect_snapshot( - gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( - repos_table = test_mocker$use("gh_repos_table_with_platform"), - verbose = TRUE, - progress = FALSE - ) + gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( + repos_table = test_mocker$use("gh_repos_table_with_platform"), + verbose = FALSE, + progress = FALSE ) expect_repos_table( gh_repos_with_contributors, diff --git a/tests/testthat/test-01-get_repos-GitLab.R b/tests/testthat/test-01-get_repos-GitLab.R index 751455f6..fffd5153 100644 --- a/tests/testthat/test-01-get_repos-GitLab.R +++ b/tests/testthat/test-01-get_repos-GitLab.R @@ -189,10 +189,9 @@ test_that("`tailor_repos_response()` tailors precisely `repos_list`", { }) test_that("REST client prepares table from GitLab repositories response", { - expect_snapshot( - gl_repos_by_code_table <- test_rest_gitlab$prepare_repos_table( - repos_list = test_mocker$use("gl_repos_by_code_tailored") - ) + gl_repos_by_code_table <- test_rest_gitlab$prepare_repos_table( + repos_list = test_mocker$use("gl_repos_by_code_tailored"), + verbose = FALSE ) expect_repos_table( gl_repos_by_code_table diff --git a/tests/testthat/test-get_commits-GitStats.R b/tests/testthat/test-get_commits-GitStats.R index aa23335f..7495250e 100644 --- a/tests/testthat/test-get_commits-GitStats.R +++ b/tests/testthat/test-get_commits-GitStats.R @@ -47,6 +47,13 @@ test_gitstats <- create_test_gitstats( inject_commits = "commits_table" ) +test_that("get_commits_stats returns error when no commits", { + test_gitstats <- create_test_gitstats() + expect_snapshot_error( + get_commits_stats(test_gitstats) + ) +}) + test_that("get_commits_stats prepares table with statistics on commits", { commits_stats <- get_commits_stats(test_gitstats) expect_s3_class(commits_stats, "commits_stats") From a02e1b548c0b67dd5581d0353048969441b8e071 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 14:42:08 +0000 Subject: [PATCH 80/94] Fix tests workflow for CRAN. --- tests/testthat/test-z-GitStats.R | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/testthat/test-z-GitStats.R b/tests/testthat/test-z-GitStats.R index 31df2b45..cff3110b 100644 --- a/tests/testthat/test-z-GitStats.R +++ b/tests/testthat/test-z-GitStats.R @@ -14,6 +14,7 @@ test_that("GitStats prints the proper info when connections are added.", { }) test_that("GitStats prints the proper info when repos are passed instead of orgs.", { + skip_on_cran() suppressMessages( test_gitstats <- create_gitstats() %>% set_github_host( @@ -107,15 +108,15 @@ test_that("show_orgs print orgs properly", { ) }) -suppressMessages( - test_gitstats <- create_gitstats() %>% - set_gitlab_host( - token = Sys.getenv("GITLAB_PAT_PUBLIC"), - orgs = "mbtests/subgroup" - ) -) - test_that("show_orgs print subgroups properly", { + skip_on_cran() + suppressMessages( + test_gitstats <- create_gitstats() %>% + set_gitlab_host( + token = Sys.getenv("GITLAB_PAT_PUBLIC"), + orgs = "mbtests/subgroup" + ) + ) expect_equal( test_gitstats$show_orgs(), "mbtests/subgroup" @@ -123,6 +124,14 @@ test_that("show_orgs print subgroups properly", { }) test_that("subgroups are cleanly printed in GitStats", { + skip_on_cran() + suppressMessages( + test_gitstats <- create_gitstats() %>% + set_gitlab_host( + token = Sys.getenv("GITLAB_PAT_PUBLIC"), + orgs = "mbtests/subgroup" + ) + ) expect_snapshot( test_gitstats ) From 8064c53457a5a6f12340c1cb1cfc99ebb151a6da Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 14:50:36 +0000 Subject: [PATCH 81/94] Add test to pass coverage package job. --- tests/testthat/test-get_commits-GitStats.R | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/testthat/test-get_commits-GitStats.R b/tests/testthat/test-get_commits-GitStats.R index 7495250e..5e7274d0 100644 --- a/tests/testthat/test-get_commits-GitStats.R +++ b/tests/testthat/test-get_commits-GitStats.R @@ -42,6 +42,24 @@ test_that("get_commits works properly", { test_mocker$cache(commits_table) }) +test_that("get_commits() works", { + mockery::stub( + get_commits, + "gitstats_object$get_commits", + test_mocker$use("commits_table") + ) + commits_table <- get_commits( + test_gitstats, + since = "2023-06-15", + until = "2023-06-30", + verbose = FALSE + ) + expect_s3_class( + commits_table, + "commits_data" + ) +}) + test_gitstats <- create_test_gitstats( hosts = 2, inject_commits = "commits_table" From 7afaf92fccd8307fb9f7d698ebcad40c22a0e1f4 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 18 Oct 2024 15:06:16 +0000 Subject: [PATCH 82/94] Add test to meet coverage job. --- tests/testthat/_snaps/get_commits-GitStats.md | 4 ++++ tests/testthat/test-get_commits-GitStats.R | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/tests/testthat/_snaps/get_commits-GitStats.md b/tests/testthat/_snaps/get_commits-GitStats.md index 9eace1d7..0963c7f0 100644 --- a/tests/testthat/_snaps/get_commits-GitStats.md +++ b/tests/testthat/_snaps/get_commits-GitStats.md @@ -1,3 +1,7 @@ +# get_commits() returns error when since is not defined + + You need to pass date to `since` parameter. + # get_commits_stats returns error when no commits x No commits found in GitStats storage. diff --git a/tests/testthat/test-get_commits-GitStats.R b/tests/testthat/test-get_commits-GitStats.R index 5e7274d0..c82e2d8d 100644 --- a/tests/testthat/test-get_commits-GitStats.R +++ b/tests/testthat/test-get_commits-GitStats.R @@ -60,6 +60,20 @@ test_that("get_commits() works", { ) }) +test_that("get_commits() returns error when since is not defined", { + mockery::stub( + get_commits, + "gitstats_object$get_commits", + test_mocker$use("commits_table") + ) + expect_snapshot_error( + get_commits( + test_gitstats, + verbose = FALSE + ) + ) +}) + test_gitstats <- create_test_gitstats( hosts = 2, inject_commits = "commits_table" From ac7cbd417b090971c5b41d85311c431cdebc7272 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 21 Oct 2024 07:21:27 +0000 Subject: [PATCH 83/94] Fix note from CRAN submission. --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 8d5645fc..f953c45c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -75,7 +75,7 @@ This is a major release with general changes in workflow (simplifying it), chang ## Simplifying workflow: -- GitStats workflow is now simplified. To pull data on `repositories`, `commits`, `R_package_usage` or other you should use directly corresponding `get_*()` functions instead of `pull_*()` which are deprecated. These `get_*()` functions pull data from API, parse it into table, add some goodies (additional columns) if needed and return table instead of `GitStats` object, which in our opinion is more intuitive and user-friendly ([#345]((https://github.com/r-world-devs/GitStats/issues/345))). That means you do not need to run in pipe two or three additional function calls as before, e.g. `pull_repos(gitstats_object) %>% get_repos() %>% get_repos_stats()`, but you just run +- GitStats workflow is now simplified. To pull data on `repositories`, `commits`, `R_package_usage` or other you should use directly corresponding `get_*()` functions instead of `pull_*()` which are deprecated. These `get_*()` functions pull data from API, parse it into table, add some goodies (additional columns) if needed and return table instead of `GitStats` object, which in our opinion is more intuitive and user-friendly ([#345](https://github.com/r-world-devs/GitStats/issues/345)). That means you do not need to run in pipe two or three additional function calls as before, e.g. `pull_repos(gitstats_object) %>% get_repos() %>% get_repos_stats()`, but you just run `get_repos(gitstats_object)` to get data you need. - Moreover, if you run for the second time `get_*()` function `GitStats` will pull the data from its storage and not from API as for the first time, unless you change parameters for the function (e.g. starting date with `since` in `get_commits()`) or change directly the `cache` parameter in the function. ([#333](https://github.com/r-world-devs/GitStats/issues/333)) - `pull_repos_contributors()` as a separate function is deprecated. The parameter `add_contributors` is now set by default to `TRUE` in `get_repos()` which seems more reasonable as user gets all the data. From 19f905e636fa92980d8b8bd92bea905ef3b58684 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 21 Oct 2024 07:29:30 +0000 Subject: [PATCH 84/94] Add test to meet coverage job. --- tests/testthat/_snaps/01-get_repos-GitHub.md | 7 +++++++ tests/testthat/test-01-get_repos-GitHub.R | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index b8230cb7..b4419cf2 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -5,3 +5,10 @@ Output [1] "\n query GetReposByOrg($login: String! $repoCursor: String!) {\n repositoryOwner(login: $login) {\n ... on Organization {\n \n repositories(first: 100 after: $repoCursor) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n \n }\n }\n }" +# `get_all_repos()` prints proper message + + Code + gh_repos_table <- github_testhost_priv$get_all_repos(verbose = TRUE) + Message + i [Host:GitHub][Engine:GraphQl][Scope:test-org] Pulling repositories... + diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index cf6eb277..3e1ccb48 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -406,6 +406,19 @@ test_that("`get_all_repos()` works as expected", { test_mocker$cache(gh_repos_table) }) +test_that("`get_all_repos()` prints proper message", { + mockery::stub( + github_testhost_priv$get_all_repos, + "graphql_engine$prepare_repos_table", + test_mocker$use("gh_repos_table") + ) + expect_snapshot( + gh_repos_table <- github_testhost_priv$get_all_repos( + verbose = TRUE + ) + ) +}) + test_that("GitHost adds `repo_api_url` column to GitHub repos table", { repos_table <- test_mocker$use("gh_repos_table") gh_repos_table_with_api_url <- github_testhost_priv$add_repo_api_url(repos_table) From deea28d49e7c835928c4b41c4fe5ad632c5f3fff Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 21 Oct 2024 07:36:56 +0000 Subject: [PATCH 85/94] Fix note from CRAN submission. --- DESCRIPTION | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 5cf3f4cc..dcafc2dc 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -7,9 +7,9 @@ Authors@R: c( person(given = "Karolina", family = "Marcinkowska", email = "karolina_marcinkowska@onet.pl", role = "aut"), person(given = "Matt", family = "Secrest", email = "secrestm@gene.com", role = "aut") ) -Description: Obtain statistics in a standardized way from multiple Git services: GitHub and GitLab for the time-being. - Its main purpose is to help teams, whose activities are spread across multiple git platforms, get their repository - metadata in a standardized way from all these platforms. +Description: Obtain statistics in a standardized way from multiple Git services. Its main purpose is to help teams, + whose activities are spread across multiple git platforms, get their repository metadata in a standardized way + from all these platforms. License: MIT + file LICENSE Encoding: UTF-8 Roxygen: list(markdown = TRUE, r6 = TRUE) From 00451d385ca4a793887e8efa0f5d9562f12b1a1c Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 21 Oct 2024 07:39:56 +0000 Subject: [PATCH 86/94] Fix version bump check job. Should not run only when DESCRIPTION is modified. --- .github/workflows/version.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/version.yaml b/.github/workflows/version.yaml index ecae3763..c41515eb 100644 --- a/.github/workflows/version.yaml +++ b/.github/workflows/version.yaml @@ -2,8 +2,7 @@ name: ⬆️ Check Version Bump on: pull_request: - paths: - - DESCRIPTION + branches: [devel, test, master] jobs: check-version: From 7ae461e27923d65118b469a04c0978bd3617e7fd Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 21 Oct 2024 07:40:56 +0000 Subject: [PATCH 87/94] Bump version. --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index dcafc2dc..7fc26d69 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.1 +Version: 2.1.1.9000 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), From 68c66296a4e38aff15744ae64057c9dc52a5efe7 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 21 Oct 2024 07:47:12 +0000 Subject: [PATCH 88/94] Add test to meet coverage job. --- tests/testthat/_snaps/get_release-GitHub.md | 8 ++++++++ tests/testthat/test-get_release-GitHub.R | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/tests/testthat/_snaps/get_release-GitHub.md b/tests/testthat/_snaps/get_release-GitHub.md index 116312b0..a46f4267 100644 --- a/tests/testthat/_snaps/get_release-GitHub.md +++ b/tests/testthat/_snaps/get_release-GitHub.md @@ -5,3 +5,11 @@ Output [1] "query GetReleasesFromRepo ($org: String!, $repo: String!) {\n repository(owner:$org, name:$repo){\n name\n url\n releases (last: 100) {\n nodes {\n name\n tagName\n publishedAt\n url\n description\n }\n }\n }\n }" +# `get_release_logs()` prints proper message when running + + Code + releases_table <- github_testhost$get_release_logs(since = "2023-05-01", until = "2023-09-30", + verbose = TRUE, progress = FALSE) + Message + i [Host:GitHub][Engine:GraphQl][Scope:test-org] Pulling release logs... + diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index 3c93c179..60a0e56b 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -67,3 +67,24 @@ test_that("`get_release_logs()` pulls release logs in the table format", { expect_lt(max(releases_table$published_at), as.POSIXct("2023-09-30")) test_mocker$cache(releases_table) }) + +test_that("`get_release_logs()` prints proper message when running", { + mockery::stub( + github_testhost$get_release_logs, + "graphql_engine$prepare_releases_table", + test_mocker$use("releases_table") + ) + mockery::stub( + github_testhost$get_release_logs, + "private$set_repositories", + test_mocker$use("repos_names") + ) + expect_snapshot( + releases_table <- github_testhost$get_release_logs( + since = "2023-05-01", + until = "2023-09-30", + verbose = TRUE, + progress = FALSE + ) + ) +}) From c31ea7b6ebf2f62bcf3a00bded647227ce54dfc4 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 21 Oct 2024 08:02:24 +0000 Subject: [PATCH 89/94] Bump version. --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7fc26d69..dcafc2dc 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.1.9000 +Version: 2.1.1 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), From 37349cecfe74a1b45df946bbd99f112956206325 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 21 Oct 2024 08:32:32 +0000 Subject: [PATCH 90/94] Set proper token for pkgdown. After fixing checks for token scopes it occured that default GITHUB_TOKEN is not sufficient. Therefore a new TEST_GITHUB_PAT was set in secret variables for the jobs. --- .github/workflows/pkgdown.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index fe54c7b4..799d9b18 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -20,8 +20,8 @@ jobs: concurrency: group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT}} + GITHUB_PAT: ${{ secrets.TEST_GITHUB_PAT }} + GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT }} permissions: contents: write steps: From 0505284e4e60da40bd3eeda3ec4e19adae8fbea2 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 21 Oct 2024 08:32:51 +0000 Subject: [PATCH 91/94] Spaces. --- .github/workflows/R-CMD-check.yaml | 4 ++-- .github/workflows/test-coverage.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 58c48806..d521844f 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -23,8 +23,8 @@ jobs: - {os: ubuntu-latest, r: 'release'} env: - GITHUB_PAT: ${{ secrets.TEST_GITHUB_PAT}} - GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT}} + GITHUB_PAT: ${{ secrets.TEST_GITHUB_PAT }} + GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT }} R_KEEP_PKG_SOURCE: yes USE_RENV: "FALSE" diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index c8e5fef1..0f815677 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest env: GITHUB_PAT: ${{ secrets.TEST_GITHUB_PAT }} - GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT}} + GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} USE_RENV: "FALSE" From 27482012a134cc9e5809ab438ed58d995c6eb6ee Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 21 Oct 2024 08:33:24 +0000 Subject: [PATCH 92/94] Bump version. --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index dcafc2dc..7fc26d69 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.1.1 +Version: 2.1.1.9000 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), From 408581f0bb14b30624058d23107974ea82fe4519 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 23 Oct 2024 07:53:31 +0000 Subject: [PATCH 93/94] Adjust DESCRIPTION to CRAN comments. Names should be put in single quotes. --- DESCRIPTION | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7fc26d69..e9191042 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,15 +1,15 @@ Package: GitStats -Title: Get Statistics from GitHub and GitLab -Version: 2.1.1.9000 +Title: Get Statistics from 'GitHub' and 'GitLab' +Version: 2.1.1.9001 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), person(given = "Karolina", family = "Marcinkowska", email = "karolina_marcinkowska@onet.pl", role = "aut"), person(given = "Matt", family = "Secrest", email = "secrestm@gene.com", role = "aut") ) -Description: Obtain statistics in a standardized way from multiple Git services. Its main purpose is to help teams, - whose activities are spread across multiple git platforms, get their repository metadata in a standardized way - from all these platforms. +Description: Obtain statistics in a standardized way from multiple 'Git' services: 'GitHub' and 'GitLab' for the time-being. + Its main purpose is to help teams, whose activities are spread across multiple git platforms, get their repository metadata + in a standardized way from all these platforms. License: MIT + file LICENSE Encoding: UTF-8 Roxygen: list(markdown = TRUE, r6 = TRUE) From b85d021b8544ebcc64c74741372d4ccc99311e0e Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 23 Oct 2024 08:03:07 +0000 Subject: [PATCH 94/94] Bump version. --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index e9191042..d9d6fd78 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from 'GitHub' and 'GitLab' -Version: 2.1.1.9001 +Version: 2.1.1 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"),