Regular Expressions

Pattern: cat
Input:   "I have a cat and a concatenation issue"
Matches: "cat" AND "cat" inside "concatenation"

grep '.' file          # Matches any line with at least 1 char
grep '\.' file         # Matches lines containing a literal dot
grep '^$' file         # Matches empty lines (nothing between ^ and $)
grep '.*' file         # Matches everything (. = any char, * = 0 or more)
grep 'c.t' file         # Matches cat, cot, cut, c1t, c_t etc.

grep '[Ee]rror' file                                       # Error or error
grep '[0-9]\{4\}' file                                     # 4 consecutive digits (BRE)
grep -E '[0-9]{4}' file                                    # same in ERE
grep '[^[:alpha:]]' file                                   # Non-letter characters
grep -E '[a-zA-Z0-9_-]+@[a-zA-Z0-9-]+\.[a-zA-Z]{2,}' file   # Email (ERE)

grep '^root' /etc/passwd       # Lines starting with "root"
grep 'bash$' /etc/passwd       # Lines ending with "bash"
grep '^$' file                 # Empty lines only
grep '^[^#]' file              # Lines NOT starting with # (non-comments)
grep -v '^#' file              # Same — invert match is cleaner
grep '^.\{80,\}$' file         # Lines longer than 80 characters (BRE)
grep -E '^.{80,}$' file        # Same in ERE

grep -P '\bcat\b' file         # Whole word "cat" (PCRE)
grep '\<cat\>' file            # GNU word boundaries (works in grep default)
grep '\<error\>' file          # Whole word "error" — not "errors" or "errored"

echo "category concatenation cat" | grep 'cat'      # Matches ALL three
echo "category concatenation cat" | grep '\<cat\>'  # Matches "cat" only

grep '^[A-Z]' file              # Lines starting with uppercase
grep '[.!?]$' file              # Lines ending with punctuation
grep -P '^\s*$' file            # Lines with only whitespace (PCRE)
grep -E '^#|^$' -v file         # Remove comments and blank lines

grep 'ab*c' file                       # BRE: zero or more b's between a and c
grep -E 'ab+c' file                    # ERE: one or more b's
grep '[0-9]\{4\}' file                 # BRE: 4-digit number
grep -E '[0-9]{4}' file                # ERE: same
grep -E '^.{80,}$' file                # Lines 80+ chars long
grep -P '\d{4}-\d{2}-\d{2}' file       # Date format YYYY-MM-DD (PCRE!)

# Apply + to a group:
grep -E '(ab)+' file             # ab, abab, ababab
grep -E '(cat|dog)+' file        # cat, dog, catdog, catcat

# Repeat a pattern:
grep -E '([0-9]{1,3}\.){3}[0-9]{1,3}' file   # IPv4 address!

# Find doubled words:
grep -E '\b(\w+)\s+\1\b' file
# Matches "the the", "is is", "error error" etc.

# In sed — swap first and second word:
sed -E 's/(\w+) (\w+)/\2 \1/' file
# "hello world" → "world hello"

grep '\|' file                 # not valid alone — see below
grep 'cat\|dog' file            # BRE: cat or dog
grep -E 'cat|dog' file          # ERE: cat or dog (cleaner)

# Multiple alternatives:
grep -E 'error|warning|critical' /var/log/syslog
grep -E '(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' file

# Alternation with anchors:
grep -E '^(root|admin|sudo)' /etc/passwd

grep '[[:digit:]]' file                    # Lines containing any digit
grep '^[[:upper:]]' file                   # Lines starting with uppercase
grep '[[:punct:]]$' file                   # Lines ending with punctuation
grep '[^[:alnum:][:space:]]' file          # Lines with special characters
grep -E '[[:xdigit:]]{32}' file            # MD5 hash (32 hex chars)
grep '[[:blank:]]' file                    # Lines containing tabs or spaces

# ==========================================
# GREP — THE KING OF LINUX REGEX TOOLS
# ==========================================

# SYNTAX:
# grep [OPTIONS] 'PATTERN' file(s)
# command | grep [OPTIONS] 'PATTERN'

# ──────────────────────────────────────────
# THREE MODES
# ──────────────────────────────────────────

grep   'pattern'  file   # BRE  — Basic Regex (default)
grep -E 'pattern' file   # ERE  — Extended Regex
grep -P 'pattern' file   # PCRE — Perl Compatible (most powerful)
egrep   'pattern' file   # Same as grep -E (deprecated but works)

# ──────────────────────────────────────────
# ESSENTIAL FLAGS
# ──────────────────────────────────────────

grep -i 'pattern' file   # Case INSENSITIVE ("Error" matches "error")
grep -v 'pattern' file   # INVERT — print lines NOT matching
grep -n 'pattern' file   # Show LINE NUMBERS
grep -c 'pattern' file   # COUNT of matching lines
grep -l 'pattern' files  # Show only FILENAMES that match
grep -L 'pattern' files  # Files that DON'T match
grep -o 'pattern' file   # Print ONLY the matched text (not whole line)
grep -w 'pattern' file   # Match WHOLE WORD only
grep -x 'pattern' file   # Match WHOLE LINE only
grep -r 'pattern' dir/   # RECURSIVE search in directory
grep -R 'pattern' dir/   # Recursive + follow symlinks
grep -A 3 'pattern' file # 3 lines AFTER match (After context)
grep -B 3 'pattern' file # 3 lines BEFORE match (Before context)
grep -C 3 'pattern' file # 3 lines before AND after (Context)
grep -m 5 'pattern' file # Stop after 5 MATCHES
grep -q 'pattern' file   # QUIET — just exit code (0=found, 1=not found)
grep -h 'pattern' files  # Suppress filename prefix in output
grep -s 'pattern' file   # Silent — suppress error messages
grep --color 'pattern' file  # Highlight matched text in color
grep -f patterns.txt file    # Read patterns FROM a file (one per line)

# ──────────────────────────────────────────
# BRE EXAMPLES
# ──────────────────────────────────────────

grep 'root'              /etc/passwd      # Lines containing "root"
grep '^root'              /etc/passwd      # Lines STARTING with "root"
grep 'nologin$'           /etc/passwd      # Lines ENDING with "nologin"
grep '^root.*bash$'       /etc/passwd      # Starts with root, ends with bash
grep '^$'                 file             # Empty lines
grep -v '^$'              file             # Non-empty lines
grep -v '^#'              /etc/hosts       # Non-comment lines
grep '[0-9]\{4\}'         file             # 4 consecutive digits (BRE)
grep '.\{80,\}'           file             # Lines 80+ chars long (BRE)
grep '\<error\>'          logfile          # Whole word "error" (GNU)
grep 'err\(or\|ant\)'     file             # "error" or "errant" (BRE group)

# ──────────────────────────────────────────
# ERE EXAMPLES (grep -E)
# ──────────────────────────────────────────

grep -E 'error|warning|critical'   syslog          # Multiple patterns (OR)
grep -E '^(root|daemon|nobody):'   /etc/passwd     # Multiple users
grep -E '[0-9]{4}-[0-9]{2}-[0-9]{2}' file          # Date YYYY-MM-DD
grep -E '[0-9]{1,3}(\.[0-9]{1,3}){3}' file         # IPv4 address
grep -E '^.{1,79}$' file                           # Lines under 80 chars
grep -E '(https?|ftp)://' file                     # URLs
grep -E '([a-z]+)\s+\1' file                       # Doubled words
grep -E '\b[A-Z]{2,}\b' file                       # ALL CAPS words
grep -E '^[[:upper:]].*[.!?]$' file                # Proper sentences

# ──────────────────────────────────────────
# PCRE EXAMPLES (grep -P)
# ──────────────────────────────────────────

grep -P '\d+' file                            # Lines with digits
grep -P '\b\w{10,}\b' file                    # Words 10+ characters long
grep -P '(?<=password: )\S+' file             # Text after "password: " (lookbehind!)
grep -P '^\s*$' file                          # Blank/whitespace-only lines
grep -P '\t' file                             # Lines containing TAB character
grep -P '[\x00-\x08\x0b\x0c\x0e-\x1f]' file   # Binary/control chars
grep -P '(?i)error' file                      # Case insensitive inline flag
grep -P '^(?!#)\S+' file                      # Non-comment non-empty lines (neg lookahead)

# ──────────────────────────────────────────
# GREP IN PIPELINES (Most Common Usage)
# ──────────────────────────────────────────

# Filter ps output:
ps aux | grep -v grep | grep nginx

# Find open ports:
ss -tlnp | grep LISTEN
netstat -tlnp | grep -E ':(80|443|22|3306)\s'

# Filter dmesg errors:
dmesg | grep -iE 'error|fail|warn|critical'

# Find running services:
systemctl list-units --type=service | grep running

# Search command history:
history | grep -E 'ssh|curl|wget' | tail -20

# Recursive code search:
grep -rn 'TODO\|FIXME\|HACK' /home/user/project/ --include="*.py"

# Count log entries per level:
grep -c 'ERROR'   /var/log/app.log
grep -c 'WARNING' /var/log/app.log

# Extract unique IPs from log:
grep -oP '\b\d{1,3}(\.\d{1,3}){3}\b' access.log | sort | uniq -c | sort -rn

# Find files containing pattern recursively:
grep -rl 'password' /etc/ 2>/dev/null

# Search multiple files, show filename + line:
grep -Hn 'pattern' *.conf

# Exclude directories:
grep -r --exclude-dir={.git,node_modules,vendor} 'pattern' ./

# ==========================================
# SED — STREAM EDITOR — REGEX SUBSTITUTION
# ==========================================

# SYNTAX:
# sed 's/FIND/REPLACE/flags' file
# sed -E 's/FIND/REPLACE/flags' file   (ERE mode)
# sed -i 's/FIND/REPLACE/flags' file   (edit in-place)
# sed -n 'p' file                      (-n suppresses auto-print)

# SUBSTITUTION FLAGS:
# g  → Replace ALL occurrences (global)
# i  → Case insensitive (GNU sed)
# p  → Print the line if substitution happened
# 2  → Replace only 2nd occurrence
# 2g → Replace from 2nd occurrence onwards

# ──────────────────────────────────────────
# BASIC SUBSTITUTION
# ──────────────────────────────────────────

sed 's/old/new/'          file   # Replace FIRST "old" per line
sed 's/old/new/g'         file   # Replace ALL "old" per line
sed 's/old/new/2'         file   # Replace only 2nd occurrence
sed 's/old/new/gi'        file   # Replace all, case insensitive
sed 's/old/new/gp'        file   # Replace all + print changed lines

# ──────────────────────────────────────────
# BRE vs ERE IN SED
# ──────────────────────────────────────────

# BRE (default) — quantifiers need backslash:
sed 's/[0-9]\+/NUM/g'     file   # Replace numbers (BRE: \+ )
sed 's/[0-9]\{4\}/YEAR/g' file   # Replace 4-digit numbers (BRE)

# ERE (sed -E) — cleaner syntax:
sed -E 's/[0-9]+/NUM/g'    file  # Replace numbers (ERE: + )
sed -E 's/[0-9]{4}/YEAR/g' file  # Replace 4-digit numbers (ERE)

# ──────────────────────────────────────────
# BACKREFERENCES IN SED — VERY POWERFUL!
# ──────────────────────────────────────────

# \1, \2 etc. reference capture groups in replacement

# Swap first and second word:
sed -E 's/(\w+) (\w+)/\2 \1/' file
# Input:  "hello world"
# Output: "world hello"

# Wrap pattern in brackets:
sed -E 's/[0-9]+/[&]/g' file
# Input:  "Port 8080 and 443"
# Output: "Port [8080] and [443]"
# & = the entire matched text

# Reformat date MM/DD/YYYY → YYYY-MM-DD:
sed -E 's|([0-9]{2})/([0-9]{2})/([0-9]{4})|\3-\1-\2|g' file
# Input:  "Date: 03/15/2024"
# Output: "Date: 2024-03-15"
# Note: | used as delimiter instead of / to avoid escaping!

# Extract usernames from /etc/passwd:
sed -E 's/([^:]+):.*/\1/' /etc/passwd
# Captures everything up to first : → just the username

# Add quotes around words:
sed -E 's/(\w+)/"\1"/g' file
# Input:  "hello world"
# Output: "\"hello\" \"world\""

# Add line numbers:
sed = file | sed 'N;s/\n/\t/'
# Prepends line number and tab to each line

# ──────────────────────────────────────────
# LINE ADDRESS SELECTION
# ──────────────────────────────────────────

sed '5s/old/new/'          file  # Only line 5
sed '1,10s/old/new/g'      file  # Lines 1-10 only
sed '5,$ s/old/new/g'      file  # Line 5 to end of file
sed '/pattern/s/old/new/g' file  # Only on lines matching pattern
sed '/start/,/end/ s/old/new/g' file  # Between matching patterns

# ──────────────────────────────────────────
# DELETE, PRINT, INSERT COMMANDS
# ──────────────────────────────────────────

sed '/pattern/d'           file  # Delete lines matching pattern
sed '/^$/d'                file  # Delete empty lines
sed '/^#/d'                file  # Delete comment lines
sed -n '/pattern/p'        file  # Print ONLY matching lines (like grep!)
sed -n '5,10p'              file  # Print lines 5 to 10
sed '/pattern/i\NEW LINE'  file  # Insert line BEFORE match
sed '/pattern/a\NEW LINE'  file  # Append line AFTER match
sed '1i\HEADER LINE'       file  # Add header at top

# ──────────────────────────────────────────
# IN-PLACE EDITING
# ──────────────────────────────────────────

sed -i 's/foo/bar/g'      file          # Edit file in-place!
sed -i.bak 's/foo/bar/g'  file          # In-place WITH backup (.bak)
sed -i 's/http:/https:/g' *.conf        # Multiple files!
sed -i '/^#/d; /^$/d'     config.txt    # Multiple commands!

# ──────────────────────────────────────────
# PRACTICAL SED EXAMPLES
# ──────────────────────────────────────────

# Remove trailing whitespace:
sed -E 's/[[:space:]]+$//' file.txt

# Remove leading whitespace:
sed -E 's/^[[:space:]]*//' file.txt

# Remove both leading and trailing whitespace:
sed -E 's/^[[:space:]]*|[[:space:]]*$//g' file.txt

# Remove HTML tags:
sed -E 's/<[^>]+>//g' page.html

# Comment out a line containing pattern:
sed -E 's/(^PermitRootLogin.*)/#\1/' /etc/ssh/sshd_config

# Uncomment lines (remove leading #):
sed 's/^#//' file

# Strip ANSI escape codes (terminal colors):
sed -E 's/\x1B\[[0-9;]*[mGKHF]//g' colored_output.txt

# Double-space a file:
sed 'G' file.txt    # G appends next line (empty) after each line

# Extract only IP addresses from file:
sed -nE 's/.*([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}).*/\1/p' file

# Replace multiple spaces with single space:
sed -E 's/ +/ /g' file.txt

# Insert blank line every N lines:
sed '3~3G' file     # Insert blank after every 3rd line

# Fix Windows line endings (CRLF → LF):
sed -i 's/\r//' file.txt

# Print lines between two patterns:
sed -n '/START/,/END/p' file.txt

# ==========================================
# AWK — PATTERN-ACTION TEXT PROCESSOR
# ==========================================

# AWK STRUCTURE:
# awk 'BEGIN{setup} /pattern/{action} END{teardown}' file

# AWK uses ERE by default.
# Fields split by whitespace: $1=first, $2=second, $NF=last

# ──────────────────────────────────────────
# PATTERN MATCHING
# ──────────────────────────────────────────

awk '/error/'           logfile       # Print lines matching "error"
awk '/^[0-9]/'          file.txt      # Lines starting with digit
awk '!/^#/'             config.txt    # Lines NOT starting with #
awk '/start/,/end/'     file.txt      # Lines between two patterns

# Match on specific field:
awk '$1 ~ /^192\.168/'  access.log    # Field 1 starts with 192.168
awk '$3 ~ /[0-9]{3}/'   file.txt      # Field 3 contains 3 digits
awk '$NF ~ /ERROR/'     logfile       # Last field contains ERROR
awk '$2 !~ /skip/'      file.txt      # Field 2 does NOT match

# ──────────────────────────────────────────
# AWK MATCH() FUNCTION
# ──────────────────────────────────────────

# match(string, regex)          → Returns index (1-based) or 0
# match(string, regex, array)   → Captures groups into array (gawk)

awk '{ if(match($0, /[0-9]+/)) print substr($0, RSTART, RLENGTH) }' file
# RSTART = start of match, RLENGTH = length of match

# Named capture with gawk:
awk '{ if(match($0, /([0-9]+)\.([0-9]+)/, a)) print a[1], a[2] }' file

# ──────────────────────────────────────────
# AWK SUB AND GSUB
# ──────────────────────────────────────────

awk '{ sub(/foo/, "bar"); print }'     file  # Replace FIRST match per line
awk '{ gsub(/foo/, "bar"); print }'    file  # Replace ALL matches
awk '{ gsub(/[0-9]+/, "NUM"); print }' file  # Replace all numbers

# Replace in specific field:
awk '{ gsub(/foo/, "bar", $2); print }' file  # Only in field 2

# ──────────────────────────────────────────
# AWK SPLIT WITH REGEX
# ──────────────────────────────────────────

# split(str, array, regex) → Split string into array
awk '{ n=split($0, a, /[,;:|]/); for(i=1;i<=n;i++) print a[i] }' file

# ──────────────────────────────────────────
# AWK WITH CUSTOM FIELD SEPARATOR
# ──────────────────────────────────────────

awk -F: '{ print $1, $3 }' /etc/passwd    # Split on colon
awk -F'\t' '{ print $2 }' file.tsv        # Split on tab
awk -F'[,;]' '{ print $1 }' file          # Split on comma OR semicolon (regex!)
awk 'BEGIN{FS=":"} { print $1 }' /etc/passwd

# ──────────────────────────────────────────
# PRACTICAL AWK EXAMPLES
# ──────────────────────────────────────────

# Print 3rd field of lines matching pattern:
awk '/error/ { print $3 }' logfile

# Sum a column (field 5) for matching lines:
awk '/200/ { sum += $5 } END { print "Total bytes:", sum }' access.log

# Print lines between two patterns:
awk '/BEGIN/,/END/ { print }' file.txt

# Count occurrences of pattern:
awk '/error/ { count++ } END { print count " errors" }' logfile

# Print lines longer than 80 chars:
awk 'length($0) > 80' file.txt

# Print unique values of a field:
awk '!seen[$1]++ { print $1 }' file.txt

# Extract key=value pairs:
awk 'match($0, /username=([^&]+)/, a) { print a[1] }' file

# Apache log analysis — count requests per IP:
awk '{ count[$1]++ } END { for(ip in count) print count[ip], ip }' access.log | sort -rn

# Print fields in reverse order:
awk '{ for(i=NF;i>=1;i--) printf "%s%s",$i,(i>1?OFS:ORS) }' file

# ==========================================
# TR — TRANSLATE OR DELETE CHARACTERS
# ==========================================

# tr works on CHARACTER SETS, not full regex patterns.
# It translates (replaces) or deletes single characters.
# It does NOT support + ? * quantifiers or anchors.

# SYNTAX:
# tr [OPTIONS] SET1 [SET2]
# command | tr SET1 SET2

# ──────────────────────────────────────────
# BASIC USAGE
# ──────────────────────────────────────────

# Translate (replace) characters:
echo "hello" | tr 'a-z' 'A-Z'           # → HELLO (lowercase → uppercase)
echo "HELLO" | tr 'A-Z' 'a-z'           # → hello (uppercase → lowercase)
echo "hello 123" | tr '0-9' '#'         # → hello ###

# Delete characters (-d flag):
echo "hello world" | tr -d ' '          # → helloworld (remove spaces)
echo "abc123" | tr -d '0-9'             # → abc (remove all digits)
echo 'He said "hi"' | tr -d '"'         # → He said hi

# Squeeze consecutive duplicates (-s flag):
echo "hello   world" | tr -s ' '        # → hello world (single space)
echo "aaabbbccc" | tr -s 'a-z'          # → abc (squeeze repeats)

# Complement (-c flag — use chars NOT in SET1):
echo "hello123" | tr -cd '0-9'          # → 123 (keep only digits)
echo "hello123" | tr -cd 'a-zA-Z'       # → hello (keep only letters)
echo "hello world" | tr -cd 'a-zA-Z\n'  # → helloworld (keep letters+newlines)

# ──────────────────────────────────────────
# POSIX CLASSES IN TR
# ──────────────────────────────────────────

echo "hello" | tr '[:lower:]' '[:upper:]'  # → HELLO
echo "HELLO" | tr '[:upper:]' '[:lower:]'  # → hello
echo "abc 123" | tr -d '[:digit:]'         # → abc  (remove digits)
echo "abc 123" | tr -d '[:alpha:]'         # →  123 (remove letters)
echo "abc 123" | tr -d '[:space:]'         # → abc123 (remove spaces)

# ──────────────────────────────────────────
# PRACTICAL TR EXAMPLES
# ──────────────────────────────────────────

# Convert Windows CRLF to Unix LF:
tr -d '\r' < windows_file.txt > unix_file.txt

# Replace newlines with spaces (join lines):
cat file.txt | tr '\n' ' '

# Convert spaces to newlines (one word per line):
echo "one two three" | tr ' ' '\n'

# Remove all non-printable characters:
tr -cd '[:print:]\n' < file.txt

# Generate random password (using /dev/urandom):
tr -dc 'A-Za-z0-9!@#$%^&*' < /dev/urandom | head -c 16; echo

# Count character frequency:
cat file.txt | tr -cs '[:alpha:]' '\n' | sort | uniq -c | sort -rn

# ==========================================
# FIND — LOCATE FILES USING REGEX
# ==========================================

# find uses Basic Regex by default.
# Use -regextype to switch to ERE or PCRE.

# SYNTAX:
# find PATH -regex 'PATTERN'
# find PATH -iregex 'PATTERN'   (case insensitive)

# ──────────────────────────────────────────
# BASIC FILE FINDING (Glob, not Regex)
# ──────────────────────────────────────────

# -name uses GLOB patterns (simpler, not full regex):
find / -name "*.php" 2>/dev/null           # All PHP files
find / -name "*.conf" 2>/dev/null          # Config files
find / -name "id_rsa*" 2>/dev/null         # SSH private keys!
find / -name "*.log" 2>/dev/null           # Log files
find / -iname "password*" 2>/dev/null      # Case-insensitive

# ──────────────────────────────────────────
# TRUE REGEX WITH FIND
# ──────────────────────────────────────────

# -regex matches the ENTIRE PATH (not just filename!)
find /etc -regex '.*\.conf'                # .conf files in /etc
find /var -regex '.*log.*\.gz'             # Compressed log files
find /home -regex '.*/\.[a-z]+'            # Hidden files (dotfiles)

# Switch to ERE (Perl) for more power:
find /etc -regextype posix-extended -regex '.*\.(conf|cfg|ini)$'
find /var -regextype posix-extended -regex '.*(error|warn|crit).*\.log'
find / -regextype posix-extended -regex '.*/(wp-config|config|settings)\.php$' 2>/dev/null

# ──────────────────────────────────────────
# COMBINING FIND WITH GREP (MOST USEFUL!)
# ──────────────────────────────────────────

# Find PHP files containing system():
find /var/www -name "*.php" -exec grep -lP 'system\s*\(' {} \;

# Find configs containing passwords:
find /etc -name "*.conf" -exec grep -iP 'password\s*=' {} /dev/null \;

# Find all files readable by world:
find / -perm -004 -type f 2>/dev/null | head -20

# Find SUID binaries (privilege escalation!):
find / -perm -4000 -type f 2>/dev/null

# Find SGID binaries:
find / -perm -2000 -type f 2>/dev/null

# Find world-writable directories:
find / -perm -002 -type d 2>/dev/null

# Find files modified in last 24 hours:
find /var/www -mtime -1 -name "*.php"

# Find recently modified files (after compromise):
find / -newer /tmp/reference_file -type f 2>/dev/null

# ==========================================
# BASH =~ OPERATOR — NATIVE REGEX IN SHELL
# ==========================================

# Bash 3.2+ supports regex with [[ ]] and =~
# Uses ERE (Extended Regular Expressions)
# Captured groups stored in: BASH_REMATCH array
# BASH_REMATCH    = full match
# BASH_REMATCH[1] = first capture group, etc.

# SYNTAX:
# if [[ "$string" =~ regex ]]; then ...

# ──────────────────────────────────────────
# BASIC MATCHING
# ──────────────────────────────────────────

# Simple match:
str="Hello World 2024"
if [[ "$str" =~ [0-9]{4} ]]; then
    echo "Contains 4-digit number: ${BASH_REMATCH}"
fi

# Case insensitive (convert to lowercase first):
str_lower="${str,,}"
if [[ "$str_lower" =~ ^hello ]]; then
    echo "Starts with hello"
fi

# ──────────────────────────────────────────
# CAPTURE GROUPS WITH BASH_REMATCH
# ──────────────────────────────────────────

# Parse date:
date_str="2024-03-15"
if [[ "$date_str" =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})$ ]]; then
    year="${BASH_REMATCH[1]}"
    month="${BASH_REMATCH[2]}"
    day="${BASH_REMATCH[3]}"
    echo "Year: $year, Month: $month, Day: $day"
fi

# Parse key=value:
line="username=johndoe"
if [[ "$line" =~ ^([a-zA-Z_]+)=(.+)$ ]]; then
    key="${BASH_REMATCH[1]}"
    val="${BASH_REMATCH[2]}"
    echo "Key: $key, Value: $val"
fi

# ──────────────────────────────────────────
# VALIDATION FUNCTIONS
# ──────────────────────────────────────────

# Validate IPv4 address:
validate_ip() {
    local ip="$1"
    if [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
        IFS='.' read -ra octets <<< "$ip"
        for oct in "${octets[@]}"; do
            (( oct > 255 )) && return 1
        done
        return 0
    fi
    return 1
}

validate_ip "192.168.1.1" && echo "Valid" || echo "Invalid"
validate_ip "999.0.0.1"   && echo "Valid" || echo "Invalid"

# Validate email:
validate_email() {
    [[ "$1" =~ ^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$ ]]
}

# Validate port number:
validate_port() {
    [[ "$1" =~ ^[0-9]{1,5}$ ]] && (( $1 >= 1 && $1 <= 65535 ))
}

# Check if string is a number:
is_number() {
    [[ "$1" =~ ^-?[0-9]+(\.[0-9]+)?$ ]]
}

# Validate hostname:
validate_hostname() {
    [[ "$1" =~ ^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?)*$ ]]
}

# Check file extension:
filename="report.pdf"
if [[ "$filename" =~ \.(jpg|jpeg|png|gif|pdf)$ ]]; then
    echo "Allowed file type: ${BASH_REMATCH[1]}"
fi

# ──────────────────────────────────────────
# REGEX IN BASH PARAMETER EXPANSION
# ──────────────────────────────────────────

# Note: Bash parameter expansion uses GLOB, not regex
# But you can combine with grep:

# Get lines matching pattern from variable:
while IFS= read -r line; do
    [[ "$line" =~ ^[[:space:]]*# ]] && continue  # Skip comments
    [[ -z "$line" ]] && continue                  # Skip blank lines
    echo "$line"
done < config.txt

# ──────────────────────────────────────────
# IMPORTANT BASH REGEX GOTCHAS
# ──────────────────────────────────────────

# Store regex in variable to avoid quoting issues:
pattern='^[0-9]{4}-[0-9]{2}-[0-9]{2}$'
if [[ "$str" =~ $pattern ]]; then   # No quotes around $pattern!
    echo "Match!"
fi
# WRONG: if [[ "$str" =~ "$pattern" ]]; then  ← quotes prevent regex!

# In Bash 3.x on macOS, use BASH_REMATCH carefully
# POSIX character classes work: [[:digit:]] [[:alpha:]] etc.

" ==========================================
" VIM/NEOVIM REGEX — SEARCH AND REPLACE
" ==========================================

" SEARCH MODE (press / or ?):
/pattern         " Search forward
?pattern         " Search backward
n                " Next match
N                " Previous match
* or #           " Search word under cursor

" SUBSTITUTE COMMAND:
" :[range]s/pattern/replacement/flags
" range: % = whole file, 1,10 = lines 1-10, . = current line

:s/old/new/         " Replace first on current line
:s/old/new/g        " Replace all on current line
:%s/old/new/g       " Replace all in ENTIRE file
:%s/old/new/gc      " Replace all with CONFIRMATION
:1,10s/old/new/g    " Replace in lines 1-10
:'<,'>s/old/new/g   " Replace in visual selection

" ──────────────────────────────────────────
" VIM REGEX MODES (important!)
" ──────────────────────────────────────────

" By default vim uses its own regex — different from PCRE!
" Use \v (very magic) for ERE-like behavior:

" Without \v (BRE-like — vim default):
:%s/\([0-9]\+\)/[\1]/g    " Backslashes needed for () and +

" With \v (very magic — cleaner, ERE-like):
:%s/\v([0-9]+)/[\1]/g     " No backslashes needed!

" MAGIC LEVELS:
" \v  Very magic    → + ? | ( ) { } are special WITHOUT \
" \m  Magic         → Only . * ^ $ [ are special (default)
" \M  No magic      → Only ^ $ are special
" \V  Very no magic → No special chars (all literal)

" ──────────────────────────────────────────
" VIM REGEX SYNTAX
" ──────────────────────────────────────────

" With \v mode:
/\v<word>          " Whole word match (< = word start, > = word end)
/\v\d+             " One or more digits
/\v[A-Z]{3}        " Exactly 3 uppercase letters
/\v(cat|dog)+      " cat or dog, one or more times
/\v^\s*$           " Blank/whitespace-only lines
/\v\w+@\w+\.\w+    " Email-like pattern

" ──────────────────────────────────────────
" PRACTICAL VIM REGEX
" ──────────────────────────────────────────

" Remove trailing whitespace:
:%s/\s\+$//e

" Remove blank lines:
:g/^$/d

" Remove comment lines:
:g/^#/d

" Add quotes around words:
:%s/\v(\w+)/"\1"/g

" Reformat date:
:%s/\v(\d{2})\/(\d{2})\/(\d{4})/\3-\1-\2/g

" Delete lines matching pattern:
:g/error/d

" Delete lines NOT matching pattern:
:v/error/d    " (or :g!/error/d)

" Print lines matching pattern (to command output):
:g/pattern/p

" Swap two words:
:%s/\v(\w+) (\w+)/\2 \1/g

" Capitalize first letter of each line:
:%s/\v^(\l)/\u\1/g

" Sort lines (requires :sort):
:%sort

" Number lines:
:%s/^/\=line('.')."\t"/

" ──────────────────────────────────────────
" VIM SEARCH FLAGS
" ──────────────────────────────────────────

" In search /pattern:
/pattern\c     " Case insensitive (\c anywhere in pattern)
/pattern\C     " Case sensitive (override ignorecase setting)

" Set in vim config:
set ignorecase    " Case insensitive by default
set smartcase     " Case sensitive IF uppercase used in pattern

" Highlight all matches:
set hlsearch
:noh              " Clear highlights temporarily

# ==========================================
# REGEX FOR SECURITY & PENTESTING ON LINUX
# ==========================================

# ──────────────────────────────────────────
# CREDENTIAL HUNTING
# ──────────────────────────────────────────

# Find passwords in files:
grep -rP 'password\s*[=:]\s*\S+' /etc/ /var/www/ 2>/dev/null
grep -riE '(pass|passwd|password|pwd)\s*[=:]\s*\S+' . 2>/dev/null

# Find API keys and tokens (common patterns):
grep -rP '(api_key|apikey|api_secret|access_token|auth_token)\s*[=:]\s*["'"'"']?[A-Za-z0-9+/]{16,}' .

# Find base64-encoded data (possible credentials):
grep -oP '[A-Za-z0-9+/]{40,}={0,2}' suspicious.txt

# Find AWS keys:
grep -rP 'AKIA[0-9A-Z]{16}' . 2>/dev/null

# Find private keys:
grep -rP '-----BEGIN (RSA |EC |DSA |OPENSSH )?PRIVATE KEY' /home/ /root/ /etc/ 2>/dev/null

# Find connection strings:
grep -rP '(mysql|postgresql|mongodb|redis):\/\/[^\s"'"'"']+' . 2>/dev/null
grep -rP 'Server\s*=.*Password\s*=' . 2>/dev/null   # MSSQL

# ──────────────────────────────────────────
# LOG ANALYSIS
# ──────────────────────────────────────────

# Extract IPs from access log:
grep -oP '\b\d{1,3}(\.\d{1,3}){3}\b' /var/log/nginx/access.log | \
    sort | uniq -c | sort -rn | head -20

# Find SQL injection attempts:
grep -P "('|\"|;|--|union\s+select|drop\s+table|1=1)" /var/log/nginx/access.log -i

# Find XSS attempts:
grep -iP '(<script|javascript:|onerror=|onload=|alert\()' /var/log/nginx/access.log

# Find path traversal attempts:
grep -P '(\.\./|\.\.\\|%2e%2e)' /var/log/nginx/access.log

# Find failed SSH logins:
grep -P 'Failed password for' /var/log/auth.log | \
    grep -oP 'from \K[\d.]+' | sort | uniq -c | sort -rn

# Find successful logins:
grep -P 'Accepted (password|publickey) for' /var/log/auth.log

# Find privilege escalation (sudo):
grep -P 'sudo:.*COMMAND' /var/log/auth.log

# ──────────────────────────────────────────
# NETWORK ANALYSIS
# ──────────────────────────────────────────

# Extract all unique IPs:
grep -oP '\b(?:(?:25[0-5]|2[0-4][0-9]|[1]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[1]?[0-9][0-9]?)\b' file | sort -u

# Extract URLs:
grep -oP 'https?://[^\s"'"'"'<>]+' file | sort -u

# Extract email addresses:
grep -oP '[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}' file | sort -u

# Extract MAC addresses:
grep -oP '([0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}' file

# Extract port numbers from netstat:
ss -tlnp | grep -oP ':\K[0-9]+(?=\s)' | sort -n | uniq

# ──────────────────────────────────────────
# CTF SPECIFIC
# ──────────────────────────────────────────

# Find flag format (HTB-style):
grep -rP 'HTB\{[^}]+\}' . 2>/dev/null
grep -rP '[A-Z]{3,5}\{[A-Za-z0-9_!@#$%^&*]+\}' . 2>/dev/null

# Extract MD5 hashes:
grep -oP '\b[a-fA-F0-9]{32}\b' file

# Extract SHA1 hashes:
grep -oP '\b[a-fA-F0-9]{40}\b' file

# Extract SHA256 hashes:
grep -oP '\b[a-fA-F0-9]{64}\b' file

# Extract JWT tokens:
grep -oP 'eyJ[A-Za-z0-9_\-]+\.eyJ[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+' file

# Decode base64 strings found in files:
grep -oP '[A-Za-z0-9+/]{20,}={0,2}' file | while read b64; do
    echo "$b64" | base64 -d 2>/dev/null | strings
done

# Find executable scripts in web directory:
find /var/www -name "*.php" -exec grep -lP '(system|exec|passthru|shell_exec|popen|proc_open)\s*\(' {} \;

# Find SUID binaries:
find / -perm -4000 -type f -exec ls -la {} \; 2>/dev/null