From 71b83f95152c5fddda42bbe24fcf0b407e41c976 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sun, 26 Nov 2023 21:49:56 +0800 Subject: [PATCH 01/11] Added details anchor --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 50168a33f4a..04747792b1c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ SecLists is the security tester's companion. It's a collection of multiple types This project is maintained by [Daniel Miessler](https://danielmiessler.com/), [Jason Haddix](https://twitter.com/Jhaddix), and [g0tmi1k](https://blog.g0tmi1k.com/). - - - + + ### Install From c321cad02e586c8cf556e089f88c101a68433884 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sun, 26 Nov 2023 21:50:17 +0800 Subject: [PATCH 02/11] Added script that formats readme --- ...get-and-patch-readme-repository-details.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 .bin/get-and-patch-readme-repository-details.py diff --git a/.bin/get-and-patch-readme-repository-details.py b/.bin/get-and-patch-readme-repository-details.py new file mode 100755 index 00000000000..fedd878f9a1 --- /dev/null +++ b/.bin/get-and-patch-readme-repository-details.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 + +# If you change the commit message you need to change .github/workflows/readme-updater.yml + +import requests,re + +REPOSITORY_API="https://api.github.com/repos/%s" +REPOSITORY="danielmiessler/SecLists" +REPOSITORY_COMMITS_API="https://api.github.com/repos/%s/commits" +REPOSITORY_COMMIT_URL="https://github.com/%s/commit/%s" +DETAILS_ANCHOR="" +DETAILS_ANCHOR_REGEX=r"%s.*?%s"%(DETAILS_ANCHOR,DETAILS_ANCHOR) +COMMIT_MESSAGE="[Github Action] Automated readme update." +DETAIL_USER_NOTICE_STRING="""%s + +### Repository details + +Size of a complete clone of SecLists is currently at `%s` + +Latest [commit](%s) was made on %s by %s + +Cloning this repository should take %i-%i minutes at 5MB/s speeds. + +%s +""" + +size=requests.get(REPOSITORY_API%(REPOSITORY)).json()['size'] # Its in kb + +suffixes=['MB','GB','TB'] +final_size=[str(size),"","KB"] + +for i in suffixes: + if len(final_size[0])>3: + + final_size[1]=final_size[0][-3:]+final_size[1] + final_size[0]=final_size[0][:-3] + final_size[2]=i + +trailing_nums=list(final_size[1]) + +decimal_len=3-len(final_size[0]) +if int(trailing_nums[decimal_len])>=5: + trailing_nums[decimal_len-1]=str(int(trailing_nums[decimal_len-1])+1) + +trailing_nums=''.join(trailing_nums) +trailing_nums=trailing_nums[:decimal_len] + +final_size=final_size[0]+'.'+trailing_nums+' '+final_size[2] + +eta_lower_bound=int(str(size/5000/60).split('.')[0]) # Get whole number after decimal point +eta_upper_bound=eta_lower_bound+1 + +commit_author="" +commit_hash="" +commit_date="" + +commits=requests.get(REPOSITORY_COMMITS_API%(REPOSITORY)).json() + +for commit in commits: + if commit["commit"]["message"].startswith("Merge pull request") and commit["commit"]["committer"]=="Github": + continue + + commit_author=commit['commit']['author']['name'] + commit_hash=commit['sha'] + commit_date=commit['commit']['author']['date'].split('T')[0] + +REPOSITORY_COMMIT_URL=REPOSITORY_COMMIT_URL%(REPOSITORY,commit_hash) +DETAIL_USER_NOTICE_STRING=DETAIL_USER_NOTICE_STRING%(DETAILS_ANCHOR,final_size,REPOSITORY_COMMIT_URL,commit_date,commit_author,eta_lower_bound,eta_upper_bound,DETAILS_ANCHOR) + +readme_contents=open("README.md").read() + +if re.match(DETAILS_ANCHOR_REGEX,readme_contents,flags=re.DOTALL): + print("[!] Error: No details anchor found!") + exit(2) + +readme_contents=re.sub(DETAILS_ANCHOR_REGEX,DETAIL_USER_NOTICE_STRING,readme_contents,flags=re.DOTALL) +open("README.md","w").write(readme_contents) \ No newline at end of file From ad89f86cc0c2098ea5225a0eb091a4bd62bdb5b4 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sun, 26 Nov 2023 21:50:30 +0800 Subject: [PATCH 03/11] Added workflow file that runs on push --- .github/workflows/readme-updater.yml | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/readme-updater.yml diff --git a/.github/workflows/readme-updater.yml b/.github/workflows/readme-updater.yml new file mode 100644 index 00000000000..dbec314090e --- /dev/null +++ b/.github/workflows/readme-updater.yml @@ -0,0 +1,36 @@ +# If you change the commit message you need to change .bin/get-and-patch-readme-repository-details.py + +name: Readme updater - Updates readme with latest stats + +on: + push: + workflow_dispatch: + +jobs: + + update-readme: + runs-on: ubuntu-latest + steps: + + - name: + uses: actions/checkout@v3 + + - name: Update readme + run: .bin/get-and-patch-readme-repository-details.py + + - name: Commit files if changed + run: | + git add -N . + + if [ -z "$(git ls-files --modified)" ]; then + echo "[+] No files were changed" + else + echo "[+] Files were changed! Pushing changed..." + git add -A + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY + git config --local user.email "example@github.com" + git config --local user.name "GitHub Action" + git commit -m "[Github Action] Automated readme update." + git push + fi + From 2334436bac77edd4a9ef689297ecdd2b2227ab40 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sun, 26 Nov 2023 21:54:18 +0800 Subject: [PATCH 04/11] skip our own commits as it may trigger loop --- .bin/get-and-patch-readme-repository-details.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.bin/get-and-patch-readme-repository-details.py b/.bin/get-and-patch-readme-repository-details.py index fedd878f9a1..f7ce4fbfa56 100755 --- a/.bin/get-and-patch-readme-repository-details.py +++ b/.bin/get-and-patch-readme-repository-details.py @@ -60,6 +60,9 @@ if commit["commit"]["message"].startswith("Merge pull request") and commit["commit"]["committer"]=="Github": continue + if commit["commit"]["message"]=="[Github Action] Automated readme update." and commit["commit"]["committer"]=="GitHub Action": + continue + commit_author=commit['commit']['author']['name'] commit_hash=commit['sha'] commit_date=commit['commit']['author']['date'].split('T')[0] From 2ecfba0829c56c8a2e31967734efae6cbd9b6d1e Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sun, 26 Nov 2023 21:59:48 +0800 Subject: [PATCH 05/11] Named checkout job --- .github/workflows/readme-updater.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/readme-updater.yml b/.github/workflows/readme-updater.yml index dbec314090e..b280572f1b1 100644 --- a/.github/workflows/readme-updater.yml +++ b/.github/workflows/readme-updater.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - - name: + - name: Clone repository uses: actions/checkout@v3 - name: Update readme From 723bb37b3aa224b95a5c02ffb6a992a6c9fd22c5 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sun, 26 Nov 2023 22:00:03 +0800 Subject: [PATCH 06/11] Reformated readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 04747792b1c..099d0f84e7a 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,12 @@ SecLists is the security tester's companion. It's a collection of multiple types This project is maintained by [Daniel Miessler](https://danielmiessler.com/), [Jason Haddix](https://twitter.com/Jhaddix), and [g0tmi1k](https://blog.g0tmi1k.com/). - - - + +- - - + ### Install **Zip** From af53b844b3f7bfad95dafd450305e854ca216dc2 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sun, 26 Nov 2023 22:02:29 +0800 Subject: [PATCH 07/11] verbose logging --- .bin/get-and-patch-readme-repository-details.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.bin/get-and-patch-readme-repository-details.py b/.bin/get-and-patch-readme-repository-details.py index f7ce4fbfa56..f120f8b0a9a 100755 --- a/.bin/get-and-patch-readme-repository-details.py +++ b/.bin/get-and-patch-readme-repository-details.py @@ -4,6 +4,8 @@ import requests,re +print("[+] Readme stats updater") + REPOSITORY_API="https://api.github.com/repos/%s" REPOSITORY="danielmiessler/SecLists" REPOSITORY_COMMITS_API="https://api.github.com/repos/%s/commits" @@ -77,4 +79,6 @@ exit(2) readme_contents=re.sub(DETAILS_ANCHOR_REGEX,DETAIL_USER_NOTICE_STRING,readme_contents,flags=re.DOTALL) -open("README.md","w").write(readme_contents) \ No newline at end of file +open("README.md","w").write(readme_contents) + +print("[+] Wrote README.md!") \ No newline at end of file From b06b7c8837c90a0a7a7ecf1249745c0236a277de Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 26 Nov 2023 14:03:32 +0000 Subject: [PATCH 08/11] [Github Action] Automated readme update. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 099d0f84e7a..6e7b479d210 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,18 @@ This project is maintained by [Daniel Miessler](https://danielmiessler.com/), [J - - - + +### Repository details + +Size of a complete clone of SecLists is currently at `1.05 GB` + +Latest [commit](https://github.com/danielmiessler/SecLists/commit/3da34d777ebcf45ca9ac15e76c953e83950c52a5) was made on 2023-11-24 by Mo Langning + +Cloning this repository should take 3-4 minutes at 5MB/s speeds. + + - - - ### Install From 05affd4889f3719100572ea5a6c528d141d7e14b Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Mon, 27 Nov 2023 23:46:37 +0800 Subject: [PATCH 09/11] Removed details about commits --- .bin/get-and-patch-readme-repository-details.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.bin/get-and-patch-readme-repository-details.py b/.bin/get-and-patch-readme-repository-details.py index f120f8b0a9a..28e7e44d12f 100755 --- a/.bin/get-and-patch-readme-repository-details.py +++ b/.bin/get-and-patch-readme-repository-details.py @@ -19,8 +19,6 @@ Size of a complete clone of SecLists is currently at `%s` -Latest [commit](%s) was made on %s by %s - Cloning this repository should take %i-%i minutes at 5MB/s speeds. %s @@ -81,4 +79,4 @@ readme_contents=re.sub(DETAILS_ANCHOR_REGEX,DETAIL_USER_NOTICE_STRING,readme_contents,flags=re.DOTALL) open("README.md","w").write(readme_contents) -print("[+] Wrote README.md!") \ No newline at end of file +print("[+] Wrote README.md!") From 4a1825c274ab31e283958bd13cdab59661d0388a Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Mon, 27 Nov 2023 23:49:42 +0800 Subject: [PATCH 10/11] Removed lines that relates to fetching commit status --- ...get-and-patch-readme-repository-details.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.bin/get-and-patch-readme-repository-details.py b/.bin/get-and-patch-readme-repository-details.py index 28e7e44d12f..12f7d9d838d 100755 --- a/.bin/get-and-patch-readme-repository-details.py +++ b/.bin/get-and-patch-readme-repository-details.py @@ -50,25 +50,8 @@ eta_lower_bound=int(str(size/5000/60).split('.')[0]) # Get whole number after decimal point eta_upper_bound=eta_lower_bound+1 -commit_author="" -commit_hash="" -commit_date="" - -commits=requests.get(REPOSITORY_COMMITS_API%(REPOSITORY)).json() - -for commit in commits: - if commit["commit"]["message"].startswith("Merge pull request") and commit["commit"]["committer"]=="Github": - continue - - if commit["commit"]["message"]=="[Github Action] Automated readme update." and commit["commit"]["committer"]=="GitHub Action": - continue - - commit_author=commit['commit']['author']['name'] - commit_hash=commit['sha'] - commit_date=commit['commit']['author']['date'].split('T')[0] - REPOSITORY_COMMIT_URL=REPOSITORY_COMMIT_URL%(REPOSITORY,commit_hash) -DETAIL_USER_NOTICE_STRING=DETAIL_USER_NOTICE_STRING%(DETAILS_ANCHOR,final_size,REPOSITORY_COMMIT_URL,commit_date,commit_author,eta_lower_bound,eta_upper_bound,DETAILS_ANCHOR) +DETAIL_USER_NOTICE_STRING=DETAIL_USER_NOTICE_STRING%(DETAILS_ANCHOR,final_size,eta_lower_bound,eta_upper_bound,DETAILS_ANCHOR) readme_contents=open("README.md").read() From d9c7c0171725713be45b095bccc173a7c10603fd Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Mon, 27 Nov 2023 23:55:19 +0800 Subject: [PATCH 11/11] Completely removed commit info code --- .bin/get-and-patch-readme-repository-details.py | 6 +----- README.md | 3 --- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.bin/get-and-patch-readme-repository-details.py b/.bin/get-and-patch-readme-repository-details.py index 12f7d9d838d..cadab904fdc 100755 --- a/.bin/get-and-patch-readme-repository-details.py +++ b/.bin/get-and-patch-readme-repository-details.py @@ -8,8 +8,6 @@ REPOSITORY_API="https://api.github.com/repos/%s" REPOSITORY="danielmiessler/SecLists" -REPOSITORY_COMMITS_API="https://api.github.com/repos/%s/commits" -REPOSITORY_COMMIT_URL="https://github.com/%s/commit/%s" DETAILS_ANCHOR="" DETAILS_ANCHOR_REGEX=r"%s.*?%s"%(DETAILS_ANCHOR,DETAILS_ANCHOR) COMMIT_MESSAGE="[Github Action] Automated readme update." @@ -21,8 +19,7 @@ Cloning this repository should take %i-%i minutes at 5MB/s speeds. -%s -""" +%s""" size=requests.get(REPOSITORY_API%(REPOSITORY)).json()['size'] # Its in kb @@ -50,7 +47,6 @@ eta_lower_bound=int(str(size/5000/60).split('.')[0]) # Get whole number after decimal point eta_upper_bound=eta_lower_bound+1 -REPOSITORY_COMMIT_URL=REPOSITORY_COMMIT_URL%(REPOSITORY,commit_hash) DETAIL_USER_NOTICE_STRING=DETAIL_USER_NOTICE_STRING%(DETAILS_ANCHOR,final_size,eta_lower_bound,eta_upper_bound,DETAILS_ANCHOR) readme_contents=open("README.md").read() diff --git a/README.md b/README.md index 6e7b479d210..42e55664490 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,10 @@ This project is maintained by [Daniel Miessler](https://danielmiessler.com/), [J Size of a complete clone of SecLists is currently at `1.05 GB` -Latest [commit](https://github.com/danielmiessler/SecLists/commit/3da34d777ebcf45ca9ac15e76c953e83950c52a5) was made on 2023-11-24 by Mo Langning - Cloning this repository should take 3-4 minutes at 5MB/s speeds. - - - - ### Install