Skip to content

Commit

Permalink
Merge pull request #503 from r-world-devs/488-move-parsing-methods-fr…
Browse files Browse the repository at this point in the history 
…om-host-to-api-clients

488 move parsing methods from host to api clients
  • Loading branch information
@maciekbanas
maciekbanas authored Oct 15, 2024
2 parents 6768f63 + 9defa4c commit bb02627
Show file tree
Hide file tree
Showing 22 changed files with 502 additions and 496 deletions.
2 changes: 1 addition & 1 deletion DESCRIPTION
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Package: GitStats
Title: Get Statistics from GitHub and GitLab
Version: 2.1.0.9002
Version: 2.1.0.9003
Authors@R: c(
person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")),
person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"),
Expand Down
161 changes: 161 additions & 0 deletions R/EngineGraphQLGitHub.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,40 @@ EngineGraphQLGitHub <- R6::R6Class(
return(full_repos_list)
},

# Parses repositories list into table.
prepare_repos_table = function(repos_list) {
if (length(repos_list) > 0) {
repos_table <- purrr::map(repos_list, function(repo) {
repo$default_branch <- if (!is.null(repo$default_branch)) {
repo$default_branch$name
} else {
""
}
last_activity_at <- as.POSIXct(repo$last_activity_at)
if (length(last_activity_at) == 0) {
last_activity_at <- gts_to_posixt(repo$created_at)
}
repo$languages <- purrr::map_chr(repo$languages$nodes, ~ .$name) %>%
paste0(collapse = ", ")
repo$created_at <- gts_to_posixt(repo$created_at)
repo$issues_open <- repo$issues_open$totalCount
repo$issues_closed <- repo$issues_closed$totalCount
repo$last_activity_at <- last_activity_at
repo$organization <- repo$organization$login
repo <- data.frame(repo) %>%
dplyr::relocate(
default_branch,
.after = repo_name
)
return(repo)
}) %>%
purrr::list_rbind()
} else {
repos_table <- NULL
}
return(repos_table)
},

# Iterator over pulling commits from all repositories.
get_commits_from_repos = function(org,
repos_names,
Expand All @@ -91,6 +125,45 @@ EngineGraphQLGitHub <- R6::R6Class(
return(repos_list_with_commits)
},

# Parses repositories' list with commits into table of commits.
prepare_commits_table = function(repos_list_with_commits,
org) {
commits_table <- purrr::imap(repos_list_with_commits, function(repo, repo_name) {
commits_row <- purrr::map_dfr(repo, function(commit) {
commit_author <- commit$node$author
commit$node$author <- commit_author$name
commit$node$author_login <- if (!is.null(commit_author$user$login)) {
commit_author$user$login
} else {
NA
}
commit$node$author_name <- if (!is.null(commit_author$user$name)) {
commit_author$user$name
} else {
NA
}
commit$node$committed_date <- gts_to_posixt(commit$node$committed_date)
commit$node
})
commits_row$repository <- repo_name
commits_row
}) %>%
purrr::discard(~ length(.) == 1) %>%
purrr::list_rbind()
if (nrow(commits_table) > 0) {
commits_table <- commits_table %>%
dplyr::mutate(
organization = org,
api_url = self$gql_api_url
) %>%
dplyr::relocate(
any_of(c("author_login", "author_name")),
.after = author
)
}
return(commits_table)
},

# Pull all given files from all repositories of an organization.
get_files_from_org = function(org,
type,
Expand Down Expand Up @@ -123,6 +196,30 @@ EngineGraphQLGitHub <- R6::R6Class(
return(org_files_list)
},

# Prepare files table.
prepare_files_table = function(files_response, org, file_path) {
if (!is.null(files_response)) {
files_table <- purrr::map(files_response, function(repository) {
purrr::imap(repository, function(file_data, file_name) {
data.frame(
"repo_name" = file_data$repo_name,
"repo_id" = file_data$repo_id,
"organization" = org,
"file_path" = file_name,
"file_content" = file_data$file$text %||% NA,
"file_size" = file_data$file$byteSize,
"repo_url" = file_data$repo_url
)
}) %>%
purrr::list_rbind()
}) %>%
purrr::list_rbind()
} else {
files_table <- NULL
}
return(files_table)
},

# Pull all files from all repositories of an organization.
get_files_structure_from_org = function(org,
type,
Expand Down Expand Up @@ -152,6 +249,29 @@ EngineGraphQLGitHub <- R6::R6Class(
return(files_structure)
},

# Prepare user table.
prepare_user_table = function(user_response) {
if (!is.null(user_response$data$user)) {
user_data <- user_response$data$user
user_data[["name"]] <- user_data$name %||% ""
user_data[["starred_repos"]] <- user_data$starred_repos$totalCount
user_data[["commits"]] <- user_data$contributions$totalCommitContributions
user_data[["issues"]] <- user_data$contributions$totalIssueContributions
user_data[["pull_requests"]] <- user_data$contributions$totalPullRequestContributions
user_data[["reviews"]] <- user_data$contributions$totalPullRequestReviewContributions
user_data[["contributions"]] <- NULL
user_data[["email"]] <- user_data$email %||% ""
user_data[["location"]] <- user_data$location %||% ""
user_data[["web_url"]] <- user_data$web_url %||% ""
user_table <- tibble::as_tibble(user_data) %>%
dplyr::relocate(c(commits, issues, pull_requests, reviews),
.after = starred_repos)
} else {
user_table <- NULL
}
return(user_table)
},

# Pull release logs from organization
get_release_logs_from_org = function(repos_names, org) {
release_responses <- purrr::map(repos_names, function(repository) {
Expand All @@ -167,6 +287,47 @@ EngineGraphQLGitHub <- R6::R6Class(
}) %>%
purrr::discard(~ length(.$data$repository$releases$nodes) == 0)
return(release_responses)
},

# Prepare releases table.
prepare_releases_table = function(releases_response, org, date_from, date_until) {
if (!is.null(releases_response)) {
releases_table <-
purrr::map(releases_response, function(release) {
release_table <- purrr::map(release$data$repository$releases$nodes, function(node) {
data.frame(
release_name = node$name,
release_tag = node$tagName,
published_at = gts_to_posixt(node$publishedAt),
release_url = node$url,
release_log = node$description
)
}) %>%
purrr::list_rbind() %>%
dplyr::mutate(
repo_name = release$data$repository$name,
repo_url = release$data$repository$url
) %>%
dplyr::relocate(
repo_name, repo_url,
.before = release_name
)
return(release_table)
}) %>%
purrr::list_rbind() %>%
dplyr::filter(
published_at <= as.POSIXct(date_until)
)
if (!is.null(date_from)) {
releases_table <- releases_table %>%
dplyr::filter(
published_at >= as.POSIXct(date_from)
)
}
} else {
releases_table <- NULL
}
return(releases_table)
}
),
private = list(
Expand Down
148 changes: 148 additions & 0 deletions R/EngineGraphQLGitLab.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,44 @@ EngineGraphQLGitLab <- R6::R6Class(
return(full_repos_list)
},

# Parses repositories list into table.
prepare_repos_table = function(repos_list) {
if (length(repos_list) > 0) {
repos_table <- purrr::map(repos_list, function(repo) {
repo <- repo$node
repo$default_branch <- repo$repository$rootRef %||% ""
repo$repository <- NULL
repo$languages <- if (length(repo$languages) > 0) {
purrr::map_chr(repo$languages, ~ .$name) %>%
paste0(collapse = ", ")
} else {
""
}
repo$created_at <- gts_to_posixt(repo$created_at)
repo$issues_open <- repo$issues$opened
repo$issues_closed <- repo$issues$closed
repo$issues <- NULL
repo$last_activity_at <- as.POSIXct(repo$last_activity_at)
repo$organization <- repo$namespace$path
repo$namespace <- NULL
repo$repo_path <- NULL # temporary to close issue 338
data.frame(repo)
}) %>%
purrr::list_rbind() %>%
dplyr::relocate(
repo_url,
.after = organization
) %>%
dplyr::relocate(
default_branch,
.after = repo_name
)
} else {
repos_table <- NULL
}
return(repos_table)
},

# Pull all given files from all repositories of a group.
# This is a one query way to get all the necessary info.
# However it may fail if query is too complex (too many files in file_paths).
Expand Down Expand Up @@ -232,6 +270,49 @@ EngineGraphQLGitLab <- R6::R6Class(
}, .progress = progress)
return(org_files_list)
},

# Prepare files table.
prepare_files_table = function(files_response, org, file_path) {
if (!is.null(files_response)) {
if (private$response_prepared_by_iteration(files_response)) {
files_table <- purrr::map(files_response, function(response_data) {
purrr::map(response_data$data$project$repository$blobs$nodes, function(file) {
data.frame(
"repo_name" = response_data$data$project$name,
"repo_id" = response_data$data$project$id,
"organization" = org,
"file_path" = file$name,
"file_content" = file$rawBlob,
"file_size" = as.integer(file$size),
"repo_url" = response_data$data$project$webUrl
)
}) %>%
purrr::list_rbind()
}) %>%
purrr::list_rbind()
} else {
files_table <- purrr::map(files_response, function(project) {
purrr::map(project$repository$blobs$nodes, function(file) {
data.frame(
"repo_name" = project$name,
"repo_id" = project$id,
"organization" = org,
"file_path" = file$name,
"file_content" = file$rawBlob,
"file_size" = as.integer(file$size),
"repo_url" = project$webUrl
)
}) %>%
purrr::list_rbind()
}) %>%
purrr::list_rbind()
}
} else {
files_table <- NULL
}
return(files_table)
},

get_files_structure_from_org = function(org,
type,
repos,
Expand All @@ -258,6 +339,28 @@ EngineGraphQLGitLab <- R6::R6Class(
return(files_structure)
},

# Prepare user table.
prepare_user_table = function(user_response) {
if (!is.null(user_response$data$user)) {
user_data <- user_response$data$user
user_data$name <- user_data$name %||% ""
user_data$starred_repos <- user_data$starred_repos$count
user_data$pull_requests <- user_data$pull_requests$count
user_data$reviews <- user_data$reviews$count
user_data$email <- user_data$email %||% ""
user_data$location <- user_data$location %||% ""
user_data$web_url <- user_data$web_url %||% ""
user_table <- tibble::as_tibble(user_data) %>%
dplyr::mutate(commits = NA,
issues = NA) %>%
dplyr::relocate(c(commits, issues),
.after = starred_repos)
} else {
user_table <- NULL
}
return(user_table)
},

# Pull all releases from all repositories of an organization.
get_release_logs_from_org = function(repos_names, org) {
release_responses <- purrr::map(repos_names, function(repository) {
Expand All @@ -272,6 +375,47 @@ EngineGraphQLGitLab <- R6::R6Class(
}) %>%
purrr::discard(~ length(.$data$project$releases$nodes) == 0)
return(release_responses)
},

# Prepare releases table.
prepare_releases_table = function(releases_response, org, date_from, date_until) {
if (length(releases_response) > 0) {
releases_table <-
purrr::map(releases_response, function(release) {
release_table <- purrr::map(release$data$project$releases$nodes, function(node) {
data.frame(
release_name = node$name,
release_tag = node$tagName,
published_at = gts_to_posixt(node$releasedAt),
release_url = node$links$selfUrl,
release_log = node$description
)
}) %>%
purrr::list_rbind() %>%
dplyr::mutate(
repo_name = release$data$project$name,
repo_url = release$data$project$webUrl
) %>%
dplyr::relocate(
repo_name, repo_url,
.before = release_name
)
return(release_table)
}) %>%
purrr::list_rbind() %>%
dplyr::filter(
published_at <= as.POSIXct(date_until)
)
if (!is.null(date_from)) {
releases_table <- releases_table %>%
dplyr::filter(
published_at >= as.POSIXct(date_from)
)
}
} else {
releases_table <- NULL
}
return(releases_table)
}
),
private = list(
Expand Down Expand Up @@ -417,6 +561,10 @@ EngineGraphQLGitLab <- R6::R6Class(
"files" = files
)
return(result)
},

response_prepared_by_iteration = function(files_response) {
!all(purrr::map_lgl(files_response, ~ all(c("name", "id", "webUrl", "repository") %in% names(.))))
}
)
)
Loading

0 comments on commit bb02627

Please sign in to comment.