From 77b90daaf164f7b243bfee23ac356b4a76bcb94c Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Mon, 28 Aug 2023 14:59:02 +0200 Subject: [PATCH 01/25] Check more --- .github/workflows/check_bash_style.yaml | 29 ++++++++++++ .github/workflows/check_build.yml | 62 +++++++++++++++++++++++++ .github/workflows/check_links.yaml | 28 +++++++++++ .github/workflows/check_spelling.yaml | 21 +++++++++ 4 files changed, 140 insertions(+) create mode 100644 .github/workflows/check_bash_style.yaml create mode 100644 .github/workflows/check_build.yml create mode 100644 .github/workflows/check_links.yaml create mode 100644 .github/workflows/check_spelling.yaml diff --git a/.github/workflows/check_bash_style.yaml b/.github/workflows/check_bash_style.yaml new file mode 100644 index 00000000..7dd1db11 --- /dev/null +++ b/.github/workflows/check_bash_style.yaml @@ -0,0 +1,29 @@ +name: Check bash style + +on: + push: + pull_request: + schedule: + - cron: "0 0 16 * *" + + +jobs: + check_bash_style: + + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + steps: + + - name: Set up Git repository + uses: actions/checkout@v2 + + - name: Clone needed repos + run: | + cd .. + git clone https://github.com/richelbilderbeek/launch_nft_locally_in_singularity_from_dockerfile + git clone https://github.com/richelbilderbeek/nextflow_troubleshooting + cd - + + - name: Check bash scripts for style + run: shellcheck --external-sources *.sh diff --git a/.github/workflows/check_build.yml b/.github/workflows/check_build.yml new file mode 100644 index 00000000..4f36ab82 --- /dev/null +++ b/.github/workflows/check_build.yml @@ -0,0 +1,62 @@ +# Check if the build works +name: Check build + +on: + push: + pull_request: + schedule: + - cron: "0 0 1 * *" + + +jobs: + check_build: + + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '8' + check-latest: true + + - run: ./clone_repo.sh + + - run: ./build_nft.sh + + # 'false' means an exit code 0 + - run: ./has_build_failed.sh + + - name: Clone needed repos + run: | + cd .. + git clone https://github.com/richelbilderbeek/launch_nft_locally_in_singularity_from_dockerfile + git clone https://github.com/richelbilderbeek/nextflow_troubleshooting + cd - + + - name: Run NFT for a max time + run: timeout 10m ./run_nft.sh + + + #- uses: r-lib/actions/setup-r@v2 + + #- name: Install dependencies + # run: | + # install.packages(c("remotes", "rcmdcheck", "stringr", "testthat")) + # shell: Rscript {0} + + #- run: ./2_update_gradle.sh + + #- run: ./3_update_gradle_build.sh + + #- run: ./4_update_backend_dockerfile.sh + + #- run: ./997_build.sh + + #- run: sudo docker image ls + + # - run: ./998_run.sh diff --git a/.github/workflows/check_links.yaml b/.github/workflows/check_links.yaml new file mode 100644 index 00000000..2e575ef9 --- /dev/null +++ b/.github/workflows/check_links.yaml @@ -0,0 +1,28 @@ +name: Check links + +on: + push: + pull_request: + schedule: + - cron: "0 0 16 * *" + + +jobs: + check_links: + + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v2 + + - name: Remove folders that do not care about broken links + run: | + rm -rf DevelopmentDesign + + - uses: gaurav-nelson/github-action-markdown-link-check@v1 + + #- uses: lycheeverse/lychee-action@v1.2.0 + # with: + # fail: true diff --git a/.github/workflows/check_spelling.yaml b/.github/workflows/check_spelling.yaml new file mode 100644 index 00000000..f8b1184c --- /dev/null +++ b/.github/workflows/check_spelling.yaml @@ -0,0 +1,21 @@ +name: Check spelling + +on: + push: + pull_request: + schedule: + - cron: "0 0 16 * *" + + +jobs: + check_spelling: + + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + steps: + + - name: Set up Git repository + uses: actions/checkout@v2 + + - uses: igsekor/pyspelling-any@v0.0.2 From 2a6081c6a25aee73615fa81f9ec7e97e39484d5d Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Tue, 16 Jan 2024 10:25:49 +0100 Subject: [PATCH 02/25] Add lesson plan --- docs/index.md | 2 +- lesson_plans/20240122/README.md | 61 +++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 lesson_plans/20240122/README.md diff --git a/docs/index.md b/docs/index.md index 35a12c48..51794beb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -24,7 +24,7 @@ Next scheduled workshop
* The purpose of this site is to give an overview of the capabilities of the awk language and **to underline** some particular **strengths or disadvantages **. * The page aims to promote this tool for use in every-days research work and urges you to find solutions yourself rather than expecting ready-made ones. -* The material assumes that you are somewhat familiar with **grep**, **sed** and have some **basic** programing experience. +* The material assumes that you are somewhat familiar with **grep**, **sed** and have some **basic** programming experience. ## What is awk? --- diff --git a/lesson_plans/20240122/README.md b/lesson_plans/20240122/README.md new file mode 100644 index 00000000..08886c4a --- /dev/null +++ b/lesson_plans/20240122/README.md @@ -0,0 +1,61 @@ +# Lesson plan + +- Awk +- Teacher: Richel +- Day: Monday 2024-01-22 + +Teaching goals: + +- Learners feel that Awk is something they can do +- Learners work in small groups +- Learners have enough breaks +- Practice to use the fundamental teaching cycle (as discussed in Mike Bell's 'The + Fundamentals of Teaching': (1) prior knowledge, + (2) presentation, (3) challenge, (4) feedback, (5) repetition + +Non-goals: + +- Discuss as many Awk concepts as possible, + instead follow the group's pace +- Stick to the schedule about Awk theory, + instead follow the group's pace +- Practice Awk theory enough to master it, + instead follow the group's pace +- Tailor to advanced Awk developers, + instead follow the beginners' pace + and let the more advanced learners help +- Teach best practices, + instead show those best practices +- Use teaching materials developed by colleagues just to be nice. + Instead, use free-online teaching materials developed by others, + so that me and my colleagues do not need to maintain it +- Learners have used a free and online book they can use for future reference. + Pavlin's material is too nice! + +Negative feedback that I will enjoy and not respond to: + +- 'The course goes too slow': this is an introduction, aimed at beginners. + The more advanced learners inevitably will find it go too slow. +- 'Lectures are too short' or 'The contents should be taught': + strategic learners will prefer passive listening + over active teaching methods. Although listening is an activity that + strategic learners appreciate, it has a too small effect for transferring knowledge. + I will not feel sorry for putting my learners to work :-) +- 'Go more in-depth': this is an introduction, aimed at beginners, + so I cover the basics and we get stuff to run. I will go as deep as time + permits me. +- 'Breaks interrupt the flow': I agree it does. Still, breaks are + important. And if you feel interrupted in a flow, it is nice to feel + you like to continue + +Negative feedback that I will respond to: + +- 'The course goes too fast': this is an introduction, aimed at beginners. + If even the beginners need more time, I must slow down + +## Schedule + +In teaching cycles: + +- What is Awk? + From 76b1ddc16d64fec4698a968c1231d4e359b3c675 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Tue, 16 Jan 2024 10:41:19 +0100 Subject: [PATCH 03/25] Add picture --- docs/index.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/index.md b/docs/index.md index 51794beb..bd2e10a8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,6 +18,37 @@ Next scheduled workshop
*source: [(some years ago)](http://web.archive.org/web/20160324050308/http://awk.info/?whygawk "Way back to 2016") http://awk.info/?whygawk +```mermaid +flowchart TD + + can_use_script[Can use a script?] + can_use_sed[Can use sed?] + can_use_grep[Can use grep?] + + awk[Use Awk] + c[Use C] + sed[Use sed] + grep[Use grep] + + compiled_language[Compiled language] + scripting_language[Scripting language] + + stream_editor[Stream editor] + work_with_regular_expressions[Work with regular expressions] + + awk --> |is a| scripting_language + c --> |is a| compiled_language + sed --> |is a| stream_editor + grep --> |allows to| work_with_regular_expressions + + can_use_script --> |no| c + can_use_script --> |yes| can_use_sed + can_use_sed --> |no| awk + can_use_sed --> |yes| can_use_grep + can_use_grep --> |no| sed + can_use_grep --> |yes| grep +``` + ## Objectives --- * The material on this site **is not a complete** guide or awk manual. From 182b7e8b566d12ff22d80a7684502c1b791128d8 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Tue, 16 Jan 2024 10:52:09 +0100 Subject: [PATCH 04/25] Check spelling and links --- .github/workflows/check_links.yaml | 28 ++++++++++------- .github/workflows/check_spelling.yaml | 7 +++++ .spellcheck.yml | 45 +++++++++++++++++++++++++++ .wordlist.txt | 1 + mlc_config.json | 14 +++++++++ 5 files changed, 83 insertions(+), 12 deletions(-) create mode 100644 .spellcheck.yml create mode 100644 .wordlist.txt create mode 100644 mlc_config.json diff --git a/.github/workflows/check_links.yaml b/.github/workflows/check_links.yaml index 2e575ef9..486ce6aa 100644 --- a/.github/workflows/check_links.yaml +++ b/.github/workflows/check_links.yaml @@ -1,28 +1,32 @@ -name: Check links - on: push: pull_request: schedule: - - cron: "0 0 16 * *" + - cron: "0 0 12 * *" +name: check-links jobs: - check_links: - + check-links: runs-on: ubuntu-latest env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - steps: - uses: actions/checkout@v2 - - name: Remove folders that do not care about broken links + - name: Remove lines with false positives run: | - rm -rf DevelopmentDesign + sed -i 's/.*check_links.yaml.*//g' README.md + sed -i 's/.*check_spelling.yaml.*//g' README.md - - uses: gaurav-nelson/github-action-markdown-link-check@v1 + # If there is a valid external link that fails, + # add it to mlc_config.json + - name: External links must be checked, do not allow '0' in the 'aliveStatusCodes' of mlc_config.json + run: if [[ $(grep --regexp "[^0-9]0[^0-9]" mlc_config.json | wc --lines) == "1" ]]; then echo "FOUND"; exit 42; fi + + - name: Internal links must checked, do not allow '400' in the 'aliveStatusCodes' of mlc_config.json + run: if [[ $(grep --regexp "[^0-9]400[^0-9]" mlc_config.json | wc --lines) == "1" ]]; then echo "FOUND"; exit 42; fi - #- uses: lycheeverse/lychee-action@v1.2.0 - # with: - # fail: true + - uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + config-file: 'mlc_config.json' diff --git a/.github/workflows/check_spelling.yaml b/.github/workflows/check_spelling.yaml index f8b1184c..7ac5d49d 100644 --- a/.github/workflows/check_spelling.yaml +++ b/.github/workflows/check_spelling.yaml @@ -1,3 +1,4 @@ +# Check that there is correct spelling in all files name: Check spelling on: @@ -18,4 +19,10 @@ jobs: - name: Set up Git repository uses: actions/checkout@v2 + - name: Remove folders that do not care about spelling + run: | + rm -rf development_design + rm -rf projects + rm -rf feedback + - uses: igsekor/pyspelling-any@v0.0.2 diff --git a/.spellcheck.yml b/.spellcheck.yml new file mode 100644 index 00000000..990b94c4 --- /dev/null +++ b/.spellcheck.yml @@ -0,0 +1,45 @@ +matrix: +- name: markdown + dictionary: + wordlists: + - .wordlist.txt + output: wordlist.dic + encoding: utf-8 + pipeline: + - pyspelling.filters.markdown: + markdown_extensions: + - markdown.extensions.extra: + - pyspelling.filters.html: + comments: false + attributes: + - alt + ignores: + - ':matches(code, pre)' + - 'code' + - 'pre' + - 'blockquote' + sources: + - '**/*.md' + default_encoding: utf-8 +- name: Quarto markdown + dictionary: + wordlists: + - .wordlist.txt + output: wordlist.dic + encoding: utf-8 + pipeline: + - pyspelling.filters.markdown: + markdown_extensions: + - markdown.extensions.extra: + - pyspelling.filters.html: + comments: false + attributes: + - alt + ignores: + - ':matches(code, pre)' + - 'code' + - 'pre' + - 'blockquote' + sources: + - '**/*.md' + default_encoding: utf-8 diff --git a/.wordlist.txt b/.wordlist.txt new file mode 100644 index 00000000..63a3879f --- /dev/null +++ b/.wordlist.txt @@ -0,0 +1 @@ +Richel diff --git a/mlc_config.json b/mlc_config.json new file mode 100644 index 00000000..85cc8aec --- /dev/null +++ b/mlc_config.json @@ -0,0 +1,14 @@ +{ + "aliveStatusCodes": [200, 403, 418, 503], + "ignorePatterns": [ + { + "pattern": "^morning_session/overview/$" + }, + { + "pattern": "^https://bianca.uppmax.uu.se" + }, + { + "pattern": "^https://rackham-gui.uppmax.uu.se" + } + ] +} From dd0fcc17e61210fccc3a8c08286b72df6dc2b57f Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Tue, 16 Jan 2024 10:55:05 +0100 Subject: [PATCH 05/25] Whitelist correctly spelled words --- .wordlist.txt | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.wordlist.txt b/.wordlist.txt index 63a3879f..662029ff 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -1 +1,15 @@ -Richel +ARGC +argc +ARGV +awk +Awk +ci +Croicu +dat +FS +github +https +io +Mihai +pmitev +usr From c29d5b433ccd1decbad57909270d3f4d96ea1796 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Tue, 16 Jan 2024 12:17:33 +0100 Subject: [PATCH 06/25] Whitelist more words --- .wordlist.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.wordlist.txt b/.wordlist.txt index 662029ff..b3d6f2e1 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -13,3 +13,17 @@ io Mihai pmitev usr +csv +displaystyle +firstname +frac +gnuplot +lastname +opensource +overline +pc +txt +varepsilon +wikipage +xy + From d9548e8dd1e3004fa919c3b123d1009171f8a251 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Tue, 16 Jan 2024 12:17:55 +0100 Subject: [PATCH 07/25] Document CI files --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ef5f69f5..c219db18 100644 --- a/README.md +++ b/README.md @@ -3,4 +3,12 @@ # to-awk-or-not This repositiory serves as an auxiliary material to an gawk course/seminar web page -[https://pmitev.github.io/to-awk-or-not/](https://pmitev.github.io/to-awk-or-not/) \ No newline at end of file +[https://pmitev.github.io/to-awk-or-not/](https://pmitev.github.io/to-awk-or-not/) + +## Files used for continuous integration scripts + +Filename |Descriptions +-----------------------------------|------------------------------------------------------------------------------------------------------ +[mlc_config.json](mlc_config.json) |Configuration of the link checker +[.spellcheck.yml](.spellcheck.yml) |Configuration of the spell checker, use `pyspelling -c .spellcheck.yml` to do spellcheck locally +[.wordlist.txt](.wordlist.txt) |Whitelisted words for the spell checker, use `pyspelling -c .spellcheck.yml` to do spellcheck locally From 8a27b4ccc70ef0dfeaddb01e0752cc9f86a7f5e0 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Tue, 16 Jan 2024 12:42:14 +0100 Subject: [PATCH 08/25] Fix spelling --- .wordlist.txt | 286 +++++++++++++++++++++++ README.md | 4 +- docs/1.Simple_example.md | 2 +- docs/2.Teasing_with_grep.md | 2 +- docs/Bio/NCBI-taxonomy.md | 2 +- docs/Case_studies/List.md | 8 +- docs/Case_studies/awk-jmol.md | 2 +- docs/Case_studies/awk_gnuplot.md | 2 +- docs/Exercises/Advanced_data_analysis.md | 4 +- docs/Exercises/Difficult_data.md | 2 +- docs/Exercises/String_manipulation.md | 8 +- docs/More_awk/Command_params.md | 2 +- docs/More_awk/Input_output.md | 2 +- docs/Python_vs_awk.md | 4 +- docs/awk_bash.md | 2 +- wordlist.dic | Bin 0 -> 9632 bytes 16 files changed, 310 insertions(+), 22 deletions(-) create mode 100644 wordlist.dic diff --git a/.wordlist.txt b/.wordlist.txt index b3d6f2e1..8eccb66f 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -26,4 +26,290 @@ txt varepsilon wikipage xy +config +json +mlc +wordlist +yml +Canidae +Canis +Cerdocyon +Corsac +Dusicyon +Lycaon +Otocyon +Vulpes +chama +corsac +familiaris +latrans +macrotis +megalotis +pictus +sp +velox +vulpes +BASHing +au +datafix +www +eV +gaussian +genbank +getline +Hm +Korona +linenums +Misspelled words: +NCBI +Pavol +Pement +phylogenetic +pre +printf +rOH +RSA +taxonID +unformatted +VCF +asc +binwidth +chr +comul +CONVFMT +Dask +de +elemnts +embl +gz +hg +ianother +itol +kb +lc +musculus +Newick +nh +nohead +nokey +noytics +nq +num +phyloP +PROCINFO +quantile +quantiles +quartile +rgb +scientificNames +sprintf +taxdump +Uncomment +aaa +abe +adn +Amrei +analyse +argv +Arsenophonus +athe +atsym +backreference +Backreferences +backreferences +BDGP +bedops +bigWigToWig +bigwigtowig +Binzer +Bioawk +bioawk +Bioinformaticians +boolean +Borreliella +bp +Buitrón +bulkm +burgdorferi +bzip +CDHit +CDHIT +cdhit +CDSs +CHGCAR +clstr +cmd +cn +Codename +CoDing +Conda +consts +coord +cov +criterium +csh +decrypt +developerWorks +dgrp +douglasgscofield +dows +Drosophila +dvr +dx +edu +EF +encodeproject +execut +Fasta +fasta +FASTA +FBtr +filedata +fmax +fmin +FNR +fontawesome +Frc +freqs +funtion +FWHM +gauss +GaussView +gcd +genomic +GFF +gff +Gnuawk +goldenPath +González +grymoire +gsub +GTF +gzip'ed +Hellström +Heng +hgdownload +hl +Homebrew +html +http +ide +INDEL +indel +INDELs +INDELS +indels +inet +infile +init +integerlist +Inten +ints +ir +isnt +Jmol +kemi +Kepp +Kernighan +Kernighan's +len +Loma +MacOS +Mahesh +Martín +Matti +maxx +md +melanogaster +Mitev +mitev +Multiline +MultiZ +Murnaghan +Myxococcales +nARGC +nasoniae +nclass +nd +neds +neighbours +nfreq +nok +np +Ntypes +numpy +OFS +os +outf +outfile +overrepresentation +pallidum +Panchal +parallelisation +Pavlin +pavlin +Pavlin's +pdf +perl +permutate +PHAST +phastCons +POSCAR +preprocess +ProLiant +ps +py +quartiles +Quilmes +readthedocs +resample +Rhizobiales +rnd +rosettacode +rtl +Scofield +se +sed +soe +sparkline +ss +stackexchange +stackoverflow +str +strfunc +subsp +sutprised +sys +tcsh +tdef +tinyutils +Transcriptome +transcriptome +Treponema +tself +UCSC +ucsc +unix +unparsed +UPPMAX +uppmax +usung +uu +valueable +VASP +vcf +ver +vibmatrix +wamt +Wannier +waterX +webarchived +wget +wikibooks +xyz +molden +htm +decyphered +pertenue + diff --git a/README.md b/README.md index c219db18..8a4e3bf8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/pmitev/to-awk-or-not/master) ![ci](https://github.com/pmitev/to-awk-or-not/workflows/ci/badge.svg) + # to-awk-or-not -This repositiory serves as an auxiliary material to an gawk course/seminar web page + +This repository serves as an auxiliary material to an gawk course/seminar web page [https://pmitev.github.io/to-awk-or-not/](https://pmitev.github.io/to-awk-or-not/) diff --git a/docs/1.Simple_example.md b/docs/1.Simple_example.md index f048a22d..173c71bf 100644 --- a/docs/1.Simple_example.md +++ b/docs/1.Simple_example.md @@ -58,7 +58,7 @@ ed only when the criteria is met** - i.e. awk will print the values of columns 3 ??? "Discussion and exercises" - Can you find all "silver" coins older than 1986? One can use grep to filter the silver coins and pipe the result to awk or do it all together in awk. - - Unfortunatelly, awk does not have a way to print/address all fields after or before a selected one. How can one print all remaining fields? + - Unfortunately, awk does not have a way to print/address all fields after or before a selected one. How can one print all remaining fields? - A `TAB` separated version 'coins.tab' is more appropriate in such cases and rather common, for the same reason, in many bioinformatics file formats `gff|bed|sam|vcf`. ## What about some math? Can I manipulate or analyze the data? diff --git a/docs/2.Teasing_with_grep.md b/docs/2.Teasing_with_grep.md index f262fb23..9552296f 100644 --- a/docs/2.Teasing_with_grep.md +++ b/docs/2.Teasing_with_grep.md @@ -62,7 +62,7 @@ At the `#!awk END` awk will run **{action_E}**. Perfect to print the collected d ??? "Exercises" - Can you add a header `# metal | weight in ounces | date minted | country of origin | description` for the output of the coins older than 1986? Use this shorter `# header` in the beginning, until you get it working. - - What wil happen if you do not provide file as input to the above exercise? + - What will happen if you do not provide file as input to the above exercise? And here is the teaser ;-). diff --git a/docs/Bio/NCBI-taxonomy.md b/docs/Bio/NCBI-taxonomy.md index b930e321..d56a6cbd 100644 --- a/docs/Bio/NCBI-taxonomy.md +++ b/docs/Bio/NCBI-taxonomy.md @@ -171,7 +171,7 @@ $ ./01.tabulate-names.awk <(bzcat names.dmp.bz2) | sort -g -k 1 | bzip2 -c > na function Cap (string) { return toupper(substr(string,0,1))substr(string,2) } ``` -Note that this script will keep the last values for any match of the same ID. It appers that the database have repeated lines that does not contain complete information and the tabulated data get destroyed. To prevent this, we need to take care that any subsequent match will be ignored. +Note that this script will keep the last values for any match of the same ID. It appears that the database have repeated lines that does not contain complete information and the tabulated data get destroyed. To prevent this, we need to take care that any subsequent match will be ignored. ``` bash diff --git a/docs/Case_studies/List.md b/docs/Case_studies/List.md index 4c1e63c0..38f91452 100644 --- a/docs/Case_studies/List.md +++ b/docs/Case_studies/List.md @@ -22,9 +22,9 @@ Here is a collection of mine and contributed awk scripts. * **[Fasta file format tips](Fasta_tips.md)** _worth to know if working often with files in multi-fasta format_ * **[Multiline fasta to single line fasta](Multi2single_fasta.md)** - _single cryptic-looking line that will decriphered during the workshop_ + _single cryptic-looking line that will decyphered during the workshop_ * **[Sequence clustering with awk](Sequence_clustering.md)** - _apllication of the multiple files approach - contribution by Martín González Buitrón_ + _application of the multiple files approach - contribution by Martín González Buitrón_ * **[Substitute scientific with common species names in a phylogenetic tree file](../Bio/NCBI-taxonomy.md)** * **[Statistics on very large columns of values](../Bio/Stat-large-files.md)** * **[Manipulating and getting statistics for .vcf and .gff files](manipulating_vcf.md)** @@ -39,11 +39,11 @@ Here is a collection of mine and contributed awk scripts. ## Physics oriented * **[Dipole moment example](Dipole_moment.md)** - _simple calulations should not be difficult to code - here is an example_ + _simple calculations should not be difficult to code - here is an example_ * **[Multiple files - VASP CHGCAR difference](CHGCAR_diff.md)** _an simplified example on how to read multiple files (bzip-ed) line-by-line simultaneously to save memory_ * **[POSCAR: reorder atom types](POSCAR_reorder.md)** - _simple task creates programing nightmare_ + _simple task creates programming nightmare_ ## Primarily used as reference * **[Awk and Gnuplot](awk_gnuplot.md)** diff --git a/docs/Case_studies/awk-jmol.md b/docs/Case_studies/awk-jmol.md index c76b6cb4..eadb8ff3 100644 --- a/docs/Case_studies/awk-jmol.md +++ b/docs/Case_studies/awk-jmol.md @@ -8,7 +8,7 @@ draw ID vector (atomno=1) {x,y,z} ``` For larger molecules this quickly becomes quite a tedious work to type all this commands... so let awk write it for us. -The output is printed to the sceen and saved in file `vectors.spt` that will later run in Jmol. +The output is printed to the screen and saved in file `vectors.spt` that will later run in Jmol. ``` awk hl_lines="1" $ awk '{i++;printf ("draw v%i vector (atomno=%i) {%f,%f,%f}\n",i,i,$1,$2,$3)}' vectors.dat | tee vectors.spt diff --git a/docs/Case_studies/awk_gnuplot.md b/docs/Case_studies/awk_gnuplot.md index 26cc21e8..93b662cd 100644 --- a/docs/Case_studies/awk_gnuplot.md +++ b/docs/Case_studies/awk_gnuplot.md @@ -4,7 +4,7 @@ I have written this script a long time ago, before Gnuplot had the options to print its own variables on the plot. Nowadays, it is possible to make the fit entirely from Gnuplot, although it will be still tricky to make some decisions if you want to align some labels. -Perhaps the most valueable part is the demonstartion of simultaneous output/input to external program (Gnuplot in this case) `#!awk while ((gnu |& getline) > 0)` and for future reference. +Perhaps the most valueable part is the demonstration of simultaneous output/input to external program (Gnuplot in this case) `#!awk while ((gnu |& getline) > 0)` and for future reference. ``` awk #!/usr/bin/awk -f diff --git a/docs/Exercises/Advanced_data_analysis.md b/docs/Exercises/Advanced_data_analysis.md index b8caac14..9d5f2bf8 100644 --- a/docs/Exercises/Advanced_data_analysis.md +++ b/docs/Exercises/Advanced_data_analysis.md @@ -1,4 +1,4 @@ -# Advanced data analisys **** +# Advanced data analysis **** You are given a file with numbers on each row - 5 in this case. !!! note "data1" @@ -11,7 +11,7 @@ You are given a file with numbers on each row - 5 in this case. Then you are given 5 numbers (let's say "1, 3, 5, 6 and 7") and you want to find how many of these numbers are matching a number on each line - think like you are about to check your lottery tickets ;-) -The solution bellow is using an "assicative arrays" trick to make it easier to loop over the reference numbers. +The solution bellow is using an "associative arrays" trick to make it easier to loop over the reference numbers. ??? "Possible solution" Not very elegant but illustrates nicely a convenient use of associated arrays as list - if ($i in n) : diff --git a/docs/Exercises/Difficult_data.md b/docs/Exercises/Difficult_data.md index d59a8aa7..80eeecc8 100644 --- a/docs/Exercises/Difficult_data.md +++ b/docs/Exercises/Difficult_data.md @@ -16,7 +16,7 @@ O103.H461 O103.H462 ![input](../images/pdata2.png) -??? "Posible solutions:" +??? "Possible solutions:" ``` awk awk -F '[][,]' '{printf("O%03d.H%03d O%03d.H%03d\n",$2,$3,$2,$4)}' data ``` diff --git a/docs/Exercises/String_manipulation.md b/docs/Exercises/String_manipulation.md index 90d3854a..d61cfba5 100644 --- a/docs/Exercises/String_manipulation.md +++ b/docs/Exercises/String_manipulation.md @@ -25,7 +25,7 @@ Ali -17.200 Peter 6.000 ``` -??? "Posible solution" +??? "Possible solution" ``` awk $ awk '{printf("%-10s%7.3f\n",$1,$2) }' strfunc.dat ``` @@ -41,7 +41,7 @@ Ali -17.200 Peter 6.000 ``` -??? "Posible solution" +??? "Possible solution" ``` awk $ awk '{printf("%-7s % .3f\n",$1,$2) }' strfunc.dat ``` @@ -57,7 +57,7 @@ Can you modify the script so you get this form which preserves the original data Peter 6 ``` -??? "Posible solution" +??? "Possible solution" ``` awk $ awk '{printf("%7s %-7s\n",$1,$2) }' strfunc.dat ``` @@ -73,7 +73,7 @@ Ali-17.200 Pet+6.000 ``` -??? "Posible solution" +??? "Possible solution" ``` awk $ awk '{printf("%.3s%+.3f\n",$1,$2) }' strfunc.dat ``` diff --git a/docs/More_awk/Command_params.md b/docs/More_awk/Command_params.md index dbeb810c..94eb6dab 100644 --- a/docs/More_awk/Command_params.md +++ b/docs/More_awk/Command_params.md @@ -1,4 +1,4 @@ -# Trick to pass paramters on the command line +# Trick to pass parameters on the command line The common way to pass options and parameters to an Awk script is via the awk option `-v varname=value` diff --git a/docs/More_awk/Input_output.md b/docs/More_awk/Input_output.md index eea8577d..07945128 100644 --- a/docs/More_awk/Input_output.md +++ b/docs/More_awk/Input_output.md @@ -1,7 +1,7 @@ # Input/Output to an external program from within awk ## Reading output from external program -Awk has a way to read output from external programs. Here is an example where we will use onle the BEGIN block in order to simplify the discussion. +Awk has a way to read output from external programs. Here is an example where we will use only the BEGIN block in order to simplify the discussion. !!! example "read_ext1.awk" ``` awk diff --git a/docs/Python_vs_awk.md b/docs/Python_vs_awk.md index 4397176c..88d8042d 100644 --- a/docs/Python_vs_awk.md +++ b/docs/Python_vs_awk.md @@ -77,7 +77,7 @@ END { for (i=1;i<=im;i++) print freq[i],fc[i],ir[i] } Somehow, I think this is much more readable and easier to modify... ??? "python challenge" - If you think that this could be done **significaanly** easier in `python` than the python solution above, here is a [link to a smaller](data/H2O.log) file on which you can try your code. I will be glad to share the solution on this page. + If you think that this could be done **significantly** easier in `python` than the python solution above, here is a [link to a smaller](data/H2O.log) file on which you can try your code. I will be glad to share the solution on this page. ``` ./extract-freq.awk H2O.log @@ -136,4 +136,4 @@ Somehow, I think this is much more readable and easier to modify... if i == 0: line = next(stdin) for row in zip(*mat): print('\t'.join(row)) - ``` \ No newline at end of file + ``` diff --git a/docs/awk_bash.md b/docs/awk_bash.md index 98c32917..361e14c4 100644 --- a/docs/awk_bash.md +++ b/docs/awk_bash.md @@ -35,7 +35,7 @@ awk ' echo "===================================" ``` -Note that with the `<< EOF ... EOF` implementation needs the `$` sign needs to be escaped, otherwise bash will replace it with the content of the shell variable it refers to. This allows you to mix bash and awk in somewhat dengerous way... +Note that with the `<< EOF ... EOF` implementation needs the `$` sign needs to be escaped, otherwise bash will replace it with the content of the shell variable it refers to. This allows you to mix bash and awk in somewhat dangerous way... ``` bash #!/bin/bash diff --git a/wordlist.dic b/wordlist.dic new file mode 100644 index 0000000000000000000000000000000000000000..e0170abd981d823fc624caa6ea21429217c7e791 GIT binary patch literal 9632 zcmai)d30k}eaC;6^dxzEwk&x(3sa^LlGKpE1k#ixbSzI~Pb9rjBxP)9+QYN-EWfd& zr#wksCbS9RK-dmVlS6^FEIB|p4J1tov}{caB?S(JgqFR8r9G5T3Z;b7B~beLT|aZC z=^xb@`TOpD_x|p0zrTAmZolWX+sB&Tx;tzSj+vGh9t(r*_Oaua9)E`XKOe{a@c$Wc zFFilFV6Xc`T;#-<$h}8Jt{4${kgt9ecqJ*m3BESp{m(J4E4_`N>kR}J_IvkmK5EXN z*rPF@7m3j~U)ZBJU)-ZPUlJMD{<6q~qO-Tgd_^Ru?V88|#k$C0FweXvDl+fMij04X zBI7Sstcv`SBJ-4oJoBEaxGHifSb&eGD=v$u(hAT~)(fovEJf&Dp$MJlC_?AC;42vBI6eoS?45JJi#1A)>%@d!BSFW{wYP~U#-Y|{z;Mfur5XZtSGWBtV)q} zuTzBXnj)PWUB6`9;tWPV4H@qr@adtjMGuUBMzUy*gU6d8X`k?}VuGXBM&1HU&a z!tYIr%zud@^IxjSdbcRT=PMPN|7u0%zeW+}Zv}ZRzoQ79+Z369yCUnqP7ykr@ zDkAUiDMIfpiqN|Qq+8ym$o#h}GX5QkjJs2jad&|<<=u+#f43sz?@?s@A1T7ny^0LH zPmy`=1p)bhB7A;Ok$D;+b>@9U5e7c0$h-#>neZu4ZS-kH*87Yi^Bz(}zQ0vOzAq?3 z=ZlKa`H~`ZzN`qHuYjw}`>G=2A68`C*TDUc`fiTz{E0n()c8^V()dvw&*QkR!Kw@l z){I^?u2f(1INqNcEE<02aj1XLHjXnf0+#b~%wScXuIQYQ=Nl|Y#b8l(4VL6X1}pMu zgLV0)B8Ke76die$BJ#b}U{T&>(2@5WtjZ4*k^83x3o;$oc1fOL(2?gGtVt2n`0g64 z$}I-#@>zo`@`xeVvb@otBkwj?mk%qVhtC?!%eM>`9chA)ku62!e~BW3 zyjGEUzi%)v?=)DJ`wiCRZxrh%>@mm4g~9~dmjT?WYy25a&;MHqU-U_pKa>V7sm z1Ln(e%;2hAp@@DRg9T|AtjM-vDKD=wSd=##tjIlzIP@nK9riuNYDK=Sh$8=65rHSN ziX}M;u2K*kS1eOUDON>JgOC3Z!uEK4dwW}+G%_!CD(HMyxMAZuHu2Bj|JxUZ$8%!nSjXVb|Z0 zCydgpSE{ek&dI9~BxCvArq`AWpM~E+{Jo_lJhmEKy7TzO)Ma2H_-FZ&~Q*A zb=%X{us`g!lmpE)7|`#dU?oyjXC3xKbu8Yb;$9OUpN#uWPYp~S#C3wBP>Y zw5jHtnUWmOtxilTULgO=b{`_}n{;>VOM)&1_l`q!W z((FUyYs0YB3_=L`auExx6zl9YxvB9rzqR7G>H&Y${xrA!)*#>y4oKaH5L4FLu)W!# zg3N359!RYPLECe?^29WSV4288^+z%Yf}b0w_8SC2U#8-DeKmuO-E#y)*2+68YzcNI zz2J+vn`_DCj`ANqsvQA`onD$|-ts7Ndzw()3oyWrdPPayG5L&~yhR$OQE%%uFZN z_nf#;F0n(}xv_P(9|e-tjfJY3Z7pHZYegIQ12En*-zP0qJks>jxbV znF@|($4qK{*fl9}Hm;Q|fxI5N4J3&dLO4v&$#S-}9(vdJ(X2_E`gcBFAiN3W!a}XS zAhSHUedT!6S~m}=8LP6)J5muNi&!l&JKEde-C}?ta~8c6Irhxxx*zrjT^F{-VwBPL zRBWTgEGlV-Q!oL$fuC)6+kxBE5yS^0+A^D5Cc{+AGBPu|j9vSK@W&k;P!?-4woHU1 zmdjitSTpHmWL^usZB(UoH+EX~FdHqZZ{llX&_UHSOzhEW6RFE}<0Bh`K@UaHLJ?Dz zHq{NaPPeC$Eq|~vTw_(XS@y2YH8~Vx>mceI!IsRW;!6culkwL2I+ihs){eItO)Nkq zk0)nS*9`r3$0M~^nlTBj8LOefpn_tl?o>`lB0JU!+D)%pVMAnd)CjT@n_kB!{}z?$ zQHvBmY^!8fcTDCevX7I88XeUchXC7-HM%5?qLbJBl}q7;Yi@T_a>=4oAmk)9lUh9O zoYBYW1bvLkBpU(yOiOZfK*X9IEqYoRRdJ?h5Q#z^XdYbfcoTe*T z7SJS_OJ6^9abD7< z{LnK)G^&%uAPl;$60NFxL9`M21Jc~EBiWe7!>HAAhyA|p9IbP>QIT5@~OH8BoFMx_-v!O!NUy0QCn8k78`edFTbz1t&Gx=_~@5uv(;CFmVh27|)taES{|ODcCr) z;q#6D1|>|<cC_QQ+;%X)L9=vLwR0xjaT}rOVc6 zM94&WcPD5BjfUGMsK;X(o|ExiQp!YJ50z9H9b`Y4ESIRoFzdmIj@PEZYasm)f|%ysvE)*BXv&jRyc=w)TLa#% znRGqDq$lE(op2WOr^vz%=ZnO&)uj?&L~6IY$PSM}-^G8xAsbZnW74I{%&Z&w+i zH5u#n(4JG#orILs9riFXZmH*I^wM>SIG^;wa7Hj(Q!~4@nm^SIcG;-4>T@*xThmEr zu-n6|aH$SrC%Rr!cLP0TTXxd%w*AJYjE=DKL<}aT;(Ff3Z7S{-Y57QkrlTbqZ4ApA z$==>m*3wsM(e&OsQt4QXQ#w5L8gNk25JQo*n<$oqw@j6=jXXwz{lX7Fvx>@Z&=G-VWIzx^|>SvtumElyy?KU>tHX7!{Z)f71 zpX6w&LN!&>fIc$C0f4nAsrq|oDg>yOG-c<~l`tp~>a0nhql;#fy^s=pojt9hXEP)@ zFa}8u8x0RT>SPLTYNm9hdV2AcQ#h^CGYmDVf-6K@YC2gtU0YnJ5=nZUfZcmds%RQ; zG@V8a4h2JOIa1|#7!OY`Gg2=Y7#X(Kw4p|xh=*8tJXYlhI2Etf7TC(FwY)~}+<0}v zKNqa|CU~=SmsQQVG^a=a!8V`fMHyF5jXyP2k#f0&i+3^r(=d{)g@En%Z zq&prD2W?Ki2$YO9h`VHr0~J+5KLBX-)y_LOOoEFHyeDI^m-Krmh6@dR*P2V$8o@gC zYg3MOZPNVbf7?X2Wty0u1dSJAVxi(ruB62}bDn}-%@fw?Yq+V|a+uTI*28X~+5AVJ5 zc5s_+%cDH2akNw-+0p)Rz)DJLGHyy7=u+Th8B#6i$oYjZN0h&Q)F)Swl2HrCvcV`F+2B|)nq z?IY98VAvviwEFuBol4Y;Tys!mH(OzkB1xMJ&7_)MBi!u`VB6SsDn8iZAkTX}^d)ks zfx8oQPn|>(y6T#KGTKJ~_RJji!AxSH8{uq{8z8f{9vU6^-Q6LXy)T!H>VBoGm1Q}Z|X;RieR$yqvUnq;zLC-F!Qsy;`YiI`>! z1PZqMI;2+MhZc?v2(;VdCc&7ZPEc7>iIZ-B6UVF9N*bxwpzpQU+0yE@3Og)DKRD6! z`@Oci+aK%_0f=8S2hU9U1C7TtlT5}rw2+7E8l2OK0Y7r8GVxV-nM&!2W}Rz? zv5^(j632(+K#KdSx;bOaj&AvD++Ku?IAE>hR|`;_8{hWU+_16XZ^3cS;xjj zmip0-9HvdqAw8cNwl&RDp>6FwO~&ax5GV3jO#Hgms)YVU0=mWa!pp#NXw0F zxtON!w{cr05sMrw&n$q`3GUAD+nMAxsWx0ikzAr_fXt+~sq4%I$y8h~)KXK)#pNZ< zbdxbI*4WbsDr`L!<9bcc78~K?@7MO*KYQG##&0bTaNn%m5A!=noV&umvV?>T3P2RtX-(phmtM6}*Z~~d`L5>rw`y%?5pmh*=)+0H4Dl>C+&i?0O=-$Sh1>R}j zZ%6$8D>Uwic-H>=c%Orh?0I}?|GSY#@jBi=!1o)_e=MS*{T15pN495yABx6m|NiI6 z@b*;JYBT3a{64ea{}6rpd?|Bt@Noh1zLEF0(VvcF$VEE)7wA6maeV3Aw;+EB`HtZ< z+TKPNua4F@6ZL8T{^zIAv5@gA=;IZSTlh@fq#4+z82u;EPQ%D~WtMIi?V?COGoOu_+%bl!uIMVOkkv?qZ9g1{#KYV>RdcK8q zM$q$nu+Oico3BPX-;LhiN!w5)iY?OQ@e zVKgRv9xZ*RpU0wiORV>I@GyXjMmwtFV37+#IOFpls4=BgztYwyzBgb=lv>k z;*7r)S)UEP3z_$D`g0NQe+X|U(cS&%<6XR8 zInT!~K`*amjY}fgzfb%2h)1QX|B2|1$WKn5$EUv2&&`oO^mzcEw?EF7f5KX?XU#k` zRwDo09AbI6vW|19!97V+}|`u>>j^XPZc^Bu@G#yTs!XPS{+*VfN- cqqUzI`Hl8n$eaz Date: Tue, 16 Jan 2024 12:45:09 +0100 Subject: [PATCH 09/25] Ignore broken links --- mlc_config.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mlc_config.json b/mlc_config.json index 85cc8aec..ca99bcd4 100644 --- a/mlc_config.json +++ b/mlc_config.json @@ -2,13 +2,16 @@ "aliveStatusCodes": [200, 403, 418, 503], "ignorePatterns": [ { - "pattern": "^morning_session/overview/$" + "pattern": "^http://130.238.141.28/inet_obs.htm$" }, { - "pattern": "^https://bianca.uppmax.uu.se" + "pattern": "^http://cms.mpi.univie.ac.at/vasp/vasp/CHGCAR_file.html$" }, { - "pattern": "^https://rackham-gui.uppmax.uu.se" + "pattern": "^http://www.ark-genomics.org/events-online-training-eu-training-course/linux-and-bioinformatics$" + }, + { + "pattern": "^https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.84.6412&rep=rep1&type=pdf$" } ] } From cc9fb7cd7dbb0629b23a7138a28e91753fcca3dd Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Tue, 16 Jan 2024 12:49:26 +0100 Subject: [PATCH 10/25] Loosen link checking strictness --- mlc_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlc_config.json b/mlc_config.json index ca99bcd4..1d140c47 100644 --- a/mlc_config.json +++ b/mlc_config.json @@ -11,7 +11,7 @@ "pattern": "^http://www.ark-genomics.org/events-online-training-eu-training-course/linux-and-bioinformatics$" }, { - "pattern": "^https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.84.6412&rep=rep1&type=pdf$" + "pattern": "^https://citeseerx.ist.psu.edu/viewdoc/download" } ] } From feb1b27dbd360fa25b1b2e2e1402e525421be3ff Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Tue, 16 Jan 2024 12:51:34 +0100 Subject: [PATCH 11/25] Allow more words --- .wordlist.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.wordlist.txt b/.wordlist.txt index 8eccb66f..e0382fc5 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -311,5 +311,12 @@ molden htm decyphered pertenue +aa +bioinformatician +Bioinformatics +bioinformatics +th +Richel +Uppsala From cf4dbc128729c36c234bf41ae3bebfe1e7dbbd0f Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Tue, 16 Jan 2024 12:55:21 +0100 Subject: [PATCH 12/25] Fix spelling --- .wordlist.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.wordlist.txt b/.wordlist.txt index e0382fc5..ccb03614 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -318,5 +318,10 @@ bioinformatics th Richel Uppsala +dataset +SNP +SNPs + + From 0f0f7a00b8d9e4aaab1fe4712b4e664551e0fc11 Mon Sep 17 00:00:00 2001 From: Richel Bilderbeek Date: Wed, 17 Jan 2024 09:05:32 +0100 Subject: [PATCH 13/25] Create jekyll-gh-pages.yml --- .github/workflows/jekyll-gh-pages.yml | 51 +++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml new file mode 100644 index 00000000..9ac4742f --- /dev/null +++ b/.github/workflows/jekyll-gh-pages.yml @@ -0,0 +1,51 @@ +# Sample workflow for building and deploying a Jekyll site to GitHub Pages +name: Deploy Jekyll with GitHub Pages dependencies preinstalled + +on: + # Runs on pushes targeting the default branch + push: + branches: ["master"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + # Build job + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Pages + uses: actions/configure-pages@v4 + - name: Build with Jekyll + uses: actions/jekyll-build-pages@v1 + with: + source: ./ + destination: ./_site + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 From 3eba87bb28358b07c621a6232fc99a3f657df869 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 09:13:05 +0100 Subject: [PATCH 14/25] Make page generator work --- docs/Case_studies/awk-jmol.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/Case_studies/awk-jmol.md b/docs/Case_studies/awk-jmol.md index eadb8ff3..4ef687ae 100644 --- a/docs/Case_studies/awk-jmol.md +++ b/docs/Case_studies/awk-jmol.md @@ -10,7 +10,10 @@ draw ID vector (atomno=1) {x,y,z} For larger molecules this quickly becomes quite a tedious work to type all this commands... so let awk write it for us. The output is printed to the screen and saved in file `vectors.spt` that will later run in Jmol. -``` awk hl_lines="1" + + + +``` $ awk '{i++;printf ("draw v%i vector (atomno=%i) {%f,%f,%f}\n",i,i,$1,$2,$3)}' vectors.dat | tee vectors.spt draw v1 vector (atomno=1) {-0.500000,0.700000,0.700000} From 51d2f489c6bd2fc8563e6d34a09ba4d00112e2fe Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 09:15:39 +0100 Subject: [PATCH 15/25] Add whitespace for the page generator --- docs/Case_studies/awk-jmol.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/Case_studies/awk-jmol.md b/docs/Case_studies/awk-jmol.md index 4ef687ae..f9e00d2a 100644 --- a/docs/Case_studies/awk-jmol.md +++ b/docs/Case_studies/awk-jmol.md @@ -10,11 +10,9 @@ draw ID vector (atomno=1) {x,y,z} For larger molecules this quickly becomes quite a tedious work to type all this commands... so let awk write it for us. The output is printed to the screen and saved in file `vectors.spt` that will later run in Jmol. - - - -``` -$ awk '{i++;printf ("draw v%i vector (atomno=%i) {%f,%f,%f}\n",i,i,$1,$2,$3)}' vectors.dat | tee vectors.spt + +```awk hl_lines="1" +$ awk '{i++;printf ("draw v%i vector (atomno=%i) { %f, %f, %f}\n",i,i,$1,$2,$3) }' vectors.dat | tee vectors.spt draw v1 vector (atomno=1) {-0.500000,0.700000,0.700000} draw v2 vector (atomno=2) {0.500000,-1.000000,0.900000} From f83fcca0b8352a97deea5f39a81c580d5542bb84 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 09:17:14 +0100 Subject: [PATCH 16/25] Trigger ci.yml --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8a4e3bf8..95fd6201 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,4 @@ Filename |Descriptions [mlc_config.json](mlc_config.json) |Configuration of the link checker [.spellcheck.yml](.spellcheck.yml) |Configuration of the spell checker, use `pyspelling -c .spellcheck.yml` to do spellcheck locally [.wordlist.txt](.wordlist.txt) |Whitelisted words for the spell checker, use `pyspelling -c .spellcheck.yml` to do spellcheck locally + From b031dfc0a72534f0c0dbc889e8ef8634c21c2d05 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 09:21:52 +0100 Subject: [PATCH 17/25] Fix build, thanks https://github.com/mkdocs/mkdocs/issues/2447#issuecomment-857379688 --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 455c241e..4987d20c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,5 +11,8 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.x + - run: pip install mkdocs==1.1.2 - run: pip install mkdocs-material - run: mkdocs gh-deploy --force + + From 12f9fe71a9d9d9ce1694a251f11b76579b05d0d4 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 09:23:33 +0100 Subject: [PATCH 18/25] Revert "Fix build, thanks https://github.com/mkdocs/mkdocs/issues/2447#issuecomment-857379688" This reverts commit b031dfc0a72534f0c0dbc889e8ef8634c21c2d05. --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4987d20c..455c241e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,8 +11,5 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.x - - run: pip install mkdocs==1.1.2 - run: pip install mkdocs-material - run: mkdocs gh-deploy --force - - From 09e2a3c936788536c1ae822b460b2134dcd32bf7 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 09:26:53 +0100 Subject: [PATCH 19/25] Document solution --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 455c241e..07cb1986 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,4 +12,15 @@ jobs: with: python-version: 3.x - run: pip install mkdocs-material + + # If this fails with: + # + # remote: Permission to richelbilderbeek/to-awk-or-not.git denied to github-actions[bot]. + # + # (or a later, less usefull error message:) + # + # subprocess.CalledProcessError: Command '['git', 'push', 'origin', 'gh-pages', '--force']' returned non-zero exit status 128 + # + # See the answer at https://stackoverflow.com/a/75175628/3364162 - run: mkdocs gh-deploy --force + From 3f7c1a59c5221a0ec57cf73a8dbb8e55502653ab Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 09:28:35 +0100 Subject: [PATCH 20/25] Revert "Create jekyll-gh-pages.yml" This reverts commit 0f0f7a00b8d9e4aaab1fe4712b4e664551e0fc11. --- .github/workflows/jekyll-gh-pages.yml | 51 --------------------------- 1 file changed, 51 deletions(-) delete mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml deleted file mode 100644 index 9ac4742f..00000000 --- a/.github/workflows/jekyll-gh-pages.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy Jekyll with GitHub Pages dependencies preinstalled - -on: - # Runs on pushes targeting the default branch - push: - branches: ["master"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - # Build job - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Pages - uses: actions/configure-pages@v4 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@v1 - with: - source: ./ - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - - # Deployment job - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 From c88ac5f76a7adf2d4719824db079d6abfa71dea1 Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 10:04:10 +0100 Subject: [PATCH 21/25] Replace hardcoded owner --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07cb1986..13c3f69c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,9 @@ jobs: python-version: 3.x - run: pip install mkdocs-material + - name: replace hard-coded owner + run: find . -type f -exec sed -i 's/pmitev/${{ github.repository_owner }}/' {} \; + # If this fails with: # # remote: Permission to richelbilderbeek/to-awk-or-not.git denied to github-actions[bot]. From 796faad68c392f2c9de35c38911e16f1e4e2e14f Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 10:07:26 +0100 Subject: [PATCH 22/25] Replace hardcoded owner --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13c3f69c..96397ad7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: - run: pip install mkdocs-material - name: replace hard-coded owner - run: find . -type f -exec sed -i 's/pmitev/${{ github.repository_owner }}/' {} \; + run: find . -type f -exec sed -i 's/pmitev/${{ github.repository_owner }}/g' {} \; # If this fails with: # From c2169eb8653f6609307c2c12e43bde3cbd86be2e Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 10:09:44 +0100 Subject: [PATCH 23/25] Show owner --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96397ad7..b7fa0a94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,9 +13,12 @@ jobs: python-version: 3.x - run: pip install mkdocs-material - - name: replace hard-coded owner + - name: replace hardcoded owner to match this repo run: find . -type f -exec sed -i 's/pmitev/${{ github.repository_owner }}/g' {} \; + - name: show owner is matching this repo + run: cat README.md + # If this fails with: # # remote: Permission to richelbilderbeek/to-awk-or-not.git denied to github-actions[bot]. From bbe721bd424f54c5d16fb1d46e25db9c6d2b48cc Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 10:24:32 +0100 Subject: [PATCH 24/25] Use pmitev as owner --- .github/workflows/ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7fa0a94..07cb1986 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,12 +13,6 @@ jobs: python-version: 3.x - run: pip install mkdocs-material - - name: replace hardcoded owner to match this repo - run: find . -type f -exec sed -i 's/pmitev/${{ github.repository_owner }}/g' {} \; - - - name: show owner is matching this repo - run: cat README.md - # If this fails with: # # remote: Permission to richelbilderbeek/to-awk-or-not.git denied to github-actions[bot]. From 8726ce823b30981c27227e135f516cb2409175cd Mon Sep 17 00:00:00 2001 From: richelbilderbeek Date: Wed, 17 Jan 2024 10:27:18 +0100 Subject: [PATCH 25/25] Remove useless CI scripts (they ended up here due to a script probably) --- .github/workflows/check_bash_style.yaml | 29 ------------ .github/workflows/check_build.yml | 62 ------------------------- 2 files changed, 91 deletions(-) delete mode 100644 .github/workflows/check_bash_style.yaml delete mode 100644 .github/workflows/check_build.yml diff --git a/.github/workflows/check_bash_style.yaml b/.github/workflows/check_bash_style.yaml deleted file mode 100644 index 7dd1db11..00000000 --- a/.github/workflows/check_bash_style.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: Check bash style - -on: - push: - pull_request: - schedule: - - cron: "0 0 16 * *" - - -jobs: - check_bash_style: - - runs-on: ubuntu-latest - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - steps: - - - name: Set up Git repository - uses: actions/checkout@v2 - - - name: Clone needed repos - run: | - cd .. - git clone https://github.com/richelbilderbeek/launch_nft_locally_in_singularity_from_dockerfile - git clone https://github.com/richelbilderbeek/nextflow_troubleshooting - cd - - - - name: Check bash scripts for style - run: shellcheck --external-sources *.sh diff --git a/.github/workflows/check_build.yml b/.github/workflows/check_build.yml deleted file mode 100644 index 4f36ab82..00000000 --- a/.github/workflows/check_build.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Check if the build works -name: Check build - -on: - push: - pull_request: - schedule: - - cron: "0 0 1 * *" - - -jobs: - check_build: - - runs-on: ubuntu-latest - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '8' - check-latest: true - - - run: ./clone_repo.sh - - - run: ./build_nft.sh - - # 'false' means an exit code 0 - - run: ./has_build_failed.sh - - - name: Clone needed repos - run: | - cd .. - git clone https://github.com/richelbilderbeek/launch_nft_locally_in_singularity_from_dockerfile - git clone https://github.com/richelbilderbeek/nextflow_troubleshooting - cd - - - - name: Run NFT for a max time - run: timeout 10m ./run_nft.sh - - - #- uses: r-lib/actions/setup-r@v2 - - #- name: Install dependencies - # run: | - # install.packages(c("remotes", "rcmdcheck", "stringr", "testthat")) - # shell: Rscript {0} - - #- run: ./2_update_gradle.sh - - #- run: ./3_update_gradle_build.sh - - #- run: ./4_update_backend_dockerfile.sh - - #- run: ./997_build.sh - - #- run: sudo docker image ls - - # - run: ./998_run.sh