vdbhb59/hosts
1
0
Fork 0
mirror of https://git.bakhai.co.in/vdbhb59/hosts.git synced 2025-11-04 04:51:30 +05:30
Code Issues Projects Releases Packages Wiki Activity
hosts/hblock
vdbhb59 19d453b066 Corrupted repo recreated 
Something broke the whole repo, and now my commits are gone too it seems.
2025-07-27 16:57:28 +05:30

806 lines
33 KiB
Bash
Raw Blame History

#!/bin/sh
set -eu
export LC_ALL='C'
# Metadata.
if [ -z "${HOSTS_VERSION+x}" ]; then HOSTS_VERSION='3.5.1'; fi
if [ -z "${HOSTS_AUTHOR+x}" ]; then HOSTS_AUTHOR='FLOSSbOxIN <dev@flossboxin.org.in>'; fi
if [ -z "${HOSTS_LICENSE+x}" ]; then HOSTS_LICENSE='MIT, https://opensource.org/licenses/MIT'; fi
if [ -z "${HOSTS_REPOSITORY+x}" ]; then HOSTS_REPOSITORY='https://git.flossboxin.org.in/vdbhb59/hosts/raw/branch/main/hblock'; fi
# Emulate ksh if the shell is zsh.
if [ -n "${ZSH_VERSION-}" ]; then emulate -L ksh; fi
# Define system and user configuration directories.
if [ -z "${ETCDIR+x}" ]; then ETCDIR='/etc'; fi
if [ -z "${XDG_CONFIG_HOME+x}" ]; then XDG_CONFIG_HOME="${HOME-}/.config"; fi
# Trap various signals to remove temporary files on exit.
abort() { : "${EXIT_STATUS:=${?}}"; rm -rf -- "${TMPDIR:-${TMP:-/tmp}}/hblock.${$}."*; trap - EXIT; exit "${EXIT_STATUS:?}"; }
{ trap abort EXIT ||:; trap abort TERM ||:; trap abort INT ||:; trap abort HUP ||:; } 2>/dev/null
# For all other signals the exit status is preserved, but USR1 and USR2 will always use 10 and 12 respectively.
{ trap 'EXIT_STATUS=10; abort' USR1; trap 'EXIT_STATUS=12; abort' USR2; } 2>/dev/null
# Built-in header.
# HOSTNAME="${HOSTNAME-"$(uname -n)"}"
HOSTNAME=""
HBLOCK_HEADER_BUILTIN="$(cat <<-EOF
127.0.0.1 localhost ${HOSTNAME?}
255.255.255.255 broadcasthost
::1 localhost ${HOSTNAME?}
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
EOF
)"
# Built-in footer.
HBLOCK_FOOTER_BUILTIN=''
# Built-in sources.
HBLOCK_SOURCES_BUILTIN="$(cat <<-'EOF'
https://raw.githubusercontent.com/ignaciocastro/a-dove-is-dumb/refs/heads/main/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/adaway.org/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/adblock-nocoin-list/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/adguard-cname-trackers/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/adguard-simplified/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/awavenue-ads/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/d3host/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/dandelionsprout-nordic/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-ara/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-bul/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-ces-slk/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-deu/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-fra/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-heb/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-ind/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-ita/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-kor/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-lav/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-lit/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-nld/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-pol/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-por/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-rus/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-spa/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist-zho/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easyprivacy/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/eth-phishing-detect/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/gfrogeye-firstparty-trackers/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/hostsvn/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/kadhosts/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/matomo.org-spammers/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/mitchellkrogza-badd-boyz-hosts/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/pgl.yoyo.org/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/phishing.army/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/red.flag.domains/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/someonewhocares.org/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/spam404.com/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/stevenblack/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/turkish-ad-hosts/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-2020/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-2021/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-2022/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-2023/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-2024/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-2025/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-abuse/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-badware/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-privacy/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/urlhaus/list.txt
https://raw.githubusercontent.com/bogachenko/fuckfuckadblock/master/fuckfuckadblock.txt
# https://www.apps2sd.info/idm/filters.txt
# https://filters.adtidy.org/windows/filters/1.txt?id=1
# https://filters.adtidy.org/extension/ublock/filters/9.txt
# https://filters.adtidy.org/windows/filters/13.txt?d=13
# https://filters.adtidy.org/extension/ublock/filters/14.txt
# https://filters.adtidy.org/extension/ublock/filters/11.txt
# https://secure.fanboy.co.nz/fanboy-indian.txt
# https://secure.fanboy.co.nz/fanboy-cookiemonster.txt
# https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-online.txt
https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt
https://easylist-downloads.adblockplus.org/antiadblockfilters.txt
EOF
)"
# Built-in allowlist.
HBLOCK_ALLOWLIST_BUILTIN=''
# Built-in denylist.
HBLOCK_DENYLIST_BUILTIN="$(cat <<-'EOF'
#
EOF
)"
# Parse command line options.
optParse() {
SEP="$(printf '\037')"
while [ "${#}" -gt '0' ]; do
case "${1?}" in
# Short options that accept a value need a "*" in their pattern because they can be found in the "-A<value>" form.
'-O'*|'--output') optArgStr "${@-}"; outputFile="${optArg?}"; shift "${optShift:?}" ;;
'-H'*|'--header') optArgStr "${@-}"; headerFile="${optArg?}"; shift "${optShift:?}" ;;
'-F'*|'--footer') optArgStr "${@-}"; footerFile="${optArg?}"; shift "${optShift:?}" ;;
'-S'*|'--sources') optArgStr "${@-}"; sourcesFile="${optArg?}"; shift "${optShift:?}" ;;
'-A'*|'--allowlist') optArgStr "${@-}"; allowlistFile="${optArg?}"; shift "${optShift:?}" ;;
'-D'*|'--denylist') optArgStr "${@-}"; denylistFile="${optArg?}"; shift "${optShift:?}" ;;
'-R'*|'--redirection') optArgStr "${@-}"; redirection="${optArg?}"; shift "${optShift:?}" ;;
'-W'*|'--wrap') optArgStr "${@-}"; wrap="${optArg?}"; shift "${optShift:?}" ;;
'-T'*|'--template') optArgStr "${@-}"; template="${optArg?}"; shift "${optShift:?}" ;;
'-C'*|'--comment') optArgStr "${@-}"; comment="${optArg?}"; shift "${optShift:?}" ;;
'-l' |'--lenient'|'--no-lenient') optArgBool "${@-}"; lenient="${optArg:?}" ;;
'-r' |'--regex'|'--no-regex') optArgBool "${@-}"; regex="${optArg:?}" ;;
'-f' |'--filter-subdomains'|'--no-filter-subdomains') optArgBool "${@-}"; filterSubdomains="${optArg:?}" ;;
'-n'*|'--retry') optArgStr "${@-}"; retry="${optArg?}"; shift "${optShift:?}" ;;
'-c' |'--continue'|'--no-continue') optArgBool "${@-}"; continue="${optArg:?}" ;;
'-p'*|'--parallel') optArgStr "${@-}"; parallel="${optArg?}"; shift "${optShift:?}" ;;
'-q' |'--quiet'|'--no-quiet') optArgBool "${@-}"; quiet="${optArg:?}" ;;
'-x'*|'--color') optArgStr "${@-}"; color="${optArg?}"; shift "${optShift:?}" ;;
'-v' |'--version') showVersion ;;
'-h' |'--help') showHelp ;;
# If "--" is found, the remaining positional parameters are saved and the parsing ends.
--) shift; _IFS="${IFS?}"; IFS="${SEP:?}"; POS="${POS-}${POS+${SEP:?}}${*-}"; IFS="${_IFS?}"; break ;;
# If a long option in the form "--opt=value" is found, it is split into "--opt" and "value".
--*=*) optSplitEquals "${@-}"; shift; set -- "${optName:?}" "${optArg?}" "${@-}"; continue ;;
# If an option did not match any pattern, an error is thrown.
-?|--*) optDie "Illegal option ${1:?}" ;;
# If multiple short options in the form "-AB" are found, they are split into "-A" and "-B".
-?*) optSplitShort "${@-}"; shift; set -- "${optAName:?}" "${optBName:?}" "${@-}"; continue ;;
# If a positional parameter is found, it is saved.
*) POS="${POS-}${POS+${SEP:?}}${1?}" ;;
esac
shift
done
}
optSplitShort() {
optAName="${1%"${1#??}"}"; optBName="-${1#??}"
}
optSplitEquals() {
optName="${1%="${1#--*=}"}"; optArg="${1#--*=}"
}
optArgStr() {
if [ -n "${1#??}" ] && [ "${1#--}" = "${1:?}" ]; then optArg="${1#??}"; optShift='0';
elif [ -n "${2+x}" ]; then optArg="${2-}"; optShift='1';
else optDie "No argument for ${1:?} option"; fi
}
optArgBool() {
if [ "${1#--no-}" = "${1:?}" ]; then optArg='true';
else optArg='false'; fi
}
optDie() {
printf '%s\n' "${@-}" "Try 'hblock --help' for more information" >&2
exit 2
}
# Show help and quit.
showHelp() {
printf '%s\n' "$(sed -e 's/%NL/\n/g' <<-EOF
Usage: hblock [OPTION]...
hBlock is a POSIX-compliant shell script that gets a list of domains that serve
ads, tracking scripts and malware from multiple sources and creates a hosts
file, among other formats, that prevents your system from connecting to them.
Options:
-O, --output <FILE|->, \${HBLOCK_OUTPUT_FILE}%NL
Output file location.%NL
If equals "-", it is printed to stdout.%NL
(default: ${outputFile?})%NL
-H, --header <FILE|builtin|none|->, \${HBLOCK_HEADER_FILE}%NL
File to be included at the beginning of the output file.%NL
If equals "builtin", the built-in value is used.%NL
If equals "none", an empty value is used.%NL
If equals "-", the stdin content is used.%NL
If unspecified and any of the following files exists, its content is used.%NL
\${XDG_CONFIG_HOME}/hblock/header%NL
${ETCDIR?}/hblock/header%NL
(default: ${headerFile?})%NL
-F, --footer <FILE|builtin|none|->, \${HBLOCK_FOOTER_FILE}%NL
File to be included at the end of the output file.%NL
If equals "builtin", the built-in value is used.%NL
If equals "none", an empty value is used.%NL
If equals "-", the stdin content is used.%NL
If unspecified and any of the following files exists, its content is used.%NL
\${XDG_CONFIG_HOME}/hblock/footer%NL
${ETCDIR?}/hblock/footer%NL
(default: ${footerFile?})%NL
-S, --sources <FILE|builtin|none|->, \${HBLOCK_SOURCES_FILE}%NL
File with line separated URLs used to generate the blocklist.%NL
If equals "builtin", the built-in value is used.%NL
If equals "none", an empty value is used.%NL
If equals "-", the stdin content is used.%NL
If unspecified and any of the following files exists, its content is used.%NL
\${XDG_CONFIG_HOME}/hblock/sources.list%NL
${ETCDIR?}/hblock/sources.list%NL
(default: ${sourcesFile?})%NL
-A, --allowlist <FILE|builtin|none|->, \${HBLOCK_ALLOWLIST_FILE}%NL
File with line separated entries to be removed from the blocklist.%NL
If equals "builtin", the built-in value is used.%NL
If equals "none", an empty value is used.%NL
If equals "-", the stdin content is used.%NL
If unspecified and any of the following files exists, its content is used.%NL
\${XDG_CONFIG_HOME}/hblock/allow.list%NL
${ETCDIR?}/hblock/allow.list%NL
(default: ${allowlistFile?})%NL
-D, --denylist <FILE|builtin|none|->, \${HBLOCK_DENYLIST_FILE}%NL
File with line separated entries to be added to the blocklist.%NL
If equals "builtin", the built-in value is used.%NL
If equals "none", an empty value is used.%NL
If equals "-", the stdin content is used.%NL
If unspecified and any of the following files exists, its content is used.%NL
\${XDG_CONFIG_HOME}/hblock/deny.list%NL
${ETCDIR?}/hblock/deny.list%NL
(default: ${denylistFile?})%NL
-R, --redirection <REDIRECTION>, \${HBLOCK_REDIRECTION}%NL
Redirection for all entries in the blocklist.%NL
(default: ${redirection?})%NL
-W, --wrap <NUMBER>, \${HBLOCK_WRAP}%NL
Break blocklist lines after this number of entries.%NL
(default: ${wrap?})%NL
-T, --template <TEMPLATE>, \${HBLOCK_TEMPLATE}%NL
Template applied to each entry.%NL
%D = <DOMAIN>, %R = <REDIRECTION>%NL
(default: ${template?})%NL
-C, --comment <COMMENT>, \${HBLOCK_COMMENT}%NL
Character used for comments.%NL
(default: ${comment?})%NL
-l, --[no-]lenient, \${HBLOCK_LENIENT}%NL
Match all entries from sources regardless of their IP, instead of
0.0.0.0, 127.0.0.1, ::, ::1 or nothing.%NL
(default: ${lenient?})%NL
-r, --[no-]regex, \${HBLOCK_REGEX}%NL
Use POSIX BREs in the allowlist instead of fixed strings.%NL
(default: ${regex?})%NL
-f, --[no-]filter-subdomains, \${HBLOCK_FILTER_SUBDOMAINS}%NL
Do not include subdomains when the parent domain is also blocked.
Useful for reducing the blocklist size in cases such as when DNS blocking
makes these subdomains redundant.%NL
(default: ${filterSubdomains?})%NL
-n, --retry <NUMBER>, \${HBLOCK_RETRY}%NL
Number of times to retry a failed download.%NL
(default: ${retry?})%NL
-c, --[no-]continue, \${HBLOCK_CONTINUE}%NL
Do not abort if a download error occurs.%NL
(default: ${continue?})%NL
-p, --parallel, \${HBLOCK_PARALLEL}%NL
Maximum concurrency for parallel downloads.%NL
(default: ${parallel?})%NL
-q, --[no-]quiet, \${HBLOCK_QUIET}%NL
Suppress non-error messages.%NL
(default: ${quiet?})%NL
-x, --color <auto|true|false>, \${HBLOCK_COLOR}%NL
Colorize the output.%NL
(default: ${color?})%NL
-v, --version%NL
Show version number and quit.%NL
-h, --help%NL
Show this help and quit.
EOF
)"
exit 0
}
# Show version number and quit.
showVersion() {
printf '%s\n' "$(cat <<-EOF
hBlock ${HOSTS_VERSION:?}
Author: ${HOSTS_AUTHOR:?}
License: ${HOSTS_LICENSE:?}
Repository: ${HOSTS_REPOSITORY:?}
EOF
)"
exit 0
}
# Check if a program exists.
exists() {
# shellcheck disable=SC2230
if command -v true; then command -v -- "${1:?}"
elif eval type type; then eval type -- "${1:?}"
else which -- "${1:?}"; fi >/dev/null 2>&1
}
# Pretty print methods.
printInfo() { [ -n "${NO_STDOUT+x}" ] || printf "${COLOR_RESET-}[${COLOR_BGREEN-}INFO${COLOR_RESET-}] %s\n" "${@-}"; }
printWarn() { [ -n "${NO_STDERR+x}" ] || printf "${COLOR_RESET-}[${COLOR_BYELLOW-}WARN${COLOR_RESET-}] %s\n" "${@-}" >&2; }
printError() { [ -n "${NO_STDERR+x}" ] || printf "${COLOR_RESET-}[${COLOR_BRED-}ERROR${COLOR_RESET-}] %s\n" "${@-}" >&2; }
printList() { [ -n "${NO_STDOUT+x}" ] || printf "${COLOR_RESET-} ${COLOR_BCYAN-}*${COLOR_RESET-} %s\n" "${@-}"; }
# Print a pseudorandom string.
rand() { :& awk -v N="${!}" 'BEGIN{srand();printf("%08x%06x",rand()*2^31-1,N)}'; }
# Create a temporary directory, file or FIFO special file.
createTemp() {
# POSIX does not specify the mktemp utility, so here comes a hacky solution.
while t="${TMPDIR:-${TMP:-/tmp}}/hblock.${$}.$(rand)" && [ -e "${t:?}" ]; do sleep 1; done
(
umask 077
case "${1-}" in
'dir') mkdir -- "${t:?}" ;;
'file') touch -- "${t:?}" ;;
'fifo') mkfifo -- "${t:?}" ;;
esac
printf '%s' "${t:?}"
)
}
# Write stdin to a file.
sponge() {
spongeFile="$(createTemp 'file')"; cat > "${spongeFile:?}"
cat -- "${spongeFile:?}" > "${1:?}"; rm -f -- "${spongeFile:?}"
}
# Count files or directories in a directory.
dirCount() { [ -e "${1:?}" ] && printf '%s' "${#}" || printf '%s' '0'; }
# Print to stdout the contents of a URL.
fetchUrl() {
# If the protocol is "file://" we can omit the download and simply use cat.
if [ "${1#file://}" != "${1:?}" ]; then cat -- "${1#file://}"
else
userAgent='Mozilla/5.0 (X11; Linux x86_64; rv:115.0) Gecko/20100101 Firefox/115.0'
if exists curl; then curl -fsSL -A "${userAgent:?}" -- "${1:?}"
elif exists wget; then wget -qO- -U "${userAgent:?}" -- "${1:?}"
elif exists fetch; then fetch -qo- --user-agent="${userAgent:?}" -- "${1:?}"
else
printError 'curl, wget or fetch are required for this script'
exit 1
fi
fi
}
# Remove comments from string.
removeComments() { sed -e 's/[[:blank:]]*#.*//;/^$/d'; }
# Transform hosts file entries to domain names.
sanitizeBlocklist() {
leadingScript='s/^[[:blank:]]*//'
trailingScript='s/[[:blank:]]*\(#.*\)\{0,1\}$//'
if [ "${1:?}" = 'true' ]; then
ipv4Script='s/^\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}[[:blank:]]\{1,\}//'
ipv6Script='s/^\([0-9a-f]\{0,4\}:\)\{2,7\}[0-9a-f]\{0,4\}[[:blank:]]\{1,\}//'
else
ipv4Script='s/^\(0\)\{0,1\}\(127\)\{0,1\}\(\.[0-9]\{1,3\}\)\{3\}[[:blank:]]\{1,\}//'
ipv6Script='s/^\(0\{0,4\}:\)\{2,7\}0\{0,3\}[01]\{0,1\}[[:blank:]]\{1,\}//'
fi
domainRegex='\([0-9a-z_-]\{1,63\}\.\)\{1,\}[a-z][0-9a-z-]\{0,61\}[0-9a-z]\.\{0,1\}'
tr -d '\r' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' \
| sed -e "${leadingScript:?};${ipv4Script:?};${ipv6Script:?};${trailingScript:?}" \
| { grep -e "^${domainRegex:?}\([[:blank:]]\{1,\}${domainRegex:?}\)*$" ||:; } \
| tr -s ' \t' '\n' | sed 's/\.$//'
}
# Remove reserved Top Level Domains.
removeReservedTLDs() {
sed -e '/\.corp$/d' \
-e '/\.domain$/d' \
-e '/\.example$/d' \
-e '/\.home$/d' \
-e '/\.host$/d' \
-e '/\.invalid$/d' \
-e '/\.lan$/d' \
-e '/\.local$/d' \
-e '/\.localdomain$/d' \
-e '/\.localhost$/d' \
-e '/\.test$/d'
}
main() {
usrConfDir="${XDG_CONFIG_HOME?}/hblock"
sysConfDir="${ETCDIR?}/hblock"
# Source environment file if exists.
# shellcheck disable=SC1091
if [ -f "${usrConfDir:?}/environment" ]; then
set -a; . "${usrConfDir:?}/environment"; set +a
elif [ -f "${sysConfDir:?}/environment" ]; then
set -a; . "${sysConfDir:?}/environment"; set +a
fi
# Output file location.
outputFile="${HBLOCK_OUTPUT_FILE-"${ETCDIR?}/hosts"}"
# File to be included at the beginning of the output file.
headerFile='builtin'
if [ -n "${HBLOCK_HEADER+x}" ]; then
HBLOCK_HEADER_BUILTIN="${HBLOCK_HEADER?}"
elif [ -n "${HBLOCK_HEADER_FILE+x}" ]; then
headerFile="${HBLOCK_HEADER_FILE?}"
elif [ -f "${usrConfDir:?}/header" ]; then
headerFile="${usrConfDir:?}/header"
elif [ -f "${sysConfDir:?}/header" ]; then
headerFile="${sysConfDir:?}/header"
fi
# File to be included at the end of the output file.
footerFile='builtin'
if [ -n "${HBLOCK_FOOTER+x}" ]; then
HBLOCK_FOOTER_BUILTIN="${HBLOCK_FOOTER?}"
elif [ -n "${HBLOCK_FOOTER_FILE+x}" ]; then
footerFile="${HBLOCK_FOOTER_FILE?}"
elif [ -f "${usrConfDir:?}/footer" ]; then
footerFile="${usrConfDir:?}/footer"
elif [ -f "${sysConfDir:?}/footer" ]; then
footerFile="${sysConfDir:?}/footer"
fi
# File with line separated URLs used to generate the blocklist.
sourcesFile='builtin'
if [ -n "${HBLOCK_SOURCES+x}" ]; then
HBLOCK_SOURCES_BUILTIN="${HBLOCK_SOURCES?}"
elif [ -n "${HBLOCK_SOURCES_FILE+x}" ]; then
sourcesFile="${HBLOCK_SOURCES_FILE?}"
elif [ -f "${usrConfDir:?}/sources.list" ]; then
sourcesFile="${usrConfDir:?}/sources.list"
elif [ -f "${sysConfDir:?}/sources.list" ]; then
sourcesFile="${sysConfDir:?}/sources.list"
fi
# File with line separated entries to be removed from the blocklist.
allowlistFile='builtin'
if [ -n "${HBLOCK_ALLOWLIST+x}" ]; then
HBLOCK_ALLOWLIST_BUILTIN="${HBLOCK_ALLOWLIST?}"
elif [ -n "${HBLOCK_ALLOWLIST_FILE+x}" ]; then
allowlistFile="${HBLOCK_ALLOWLIST_FILE?}"
elif [ -f "${usrConfDir:?}/allow.list" ]; then
allowlistFile="${usrConfDir:?}/allow.list"
elif [ -f "${sysConfDir:?}/allow.list" ]; then
allowlistFile="${sysConfDir:?}/allow.list"
fi
# File with line separated entries to be added to the blocklist.
denylistFile='builtin'
if [ -n "${HBLOCK_DENYLIST+x}" ]; then
HBLOCK_DENYLIST_BUILTIN="${HBLOCK_DENYLIST?}"
elif [ -n "${HBLOCK_DENYLIST_FILE+x}" ]; then
denylistFile="${HBLOCK_DENYLIST_FILE?}"
elif [ -f "${usrConfDir:?}/deny.list" ]; then
denylistFile="${usrConfDir:?}/deny.list"
elif [ -f "${sysConfDir:?}/deny.list" ]; then
denylistFile="${sysConfDir:?}/deny.list"
fi
# Redirection for all entries in the blocklist.
redirection="${HBLOCK_REDIRECTION-"0.0.0.0"}"
# Break blocklist lines after this number of entries.
wrap="${HBLOCK_WRAP-"1"}"
# Template applied to each entry.
template="${HBLOCK_TEMPLATE-"%R %D"}"
# Character used for comments.
comment="${HBLOCK_COMMENT-"#"}"
# Match all entries from sources, regardless of their IP.
lenient="${HBLOCK_LENIENT-"false"}"
# Use POSIX BREs instead of fixed strings.
regex="${HBLOCK_REGEX-"false"}"
# Do not include subdomains when the parent domain is also blocked.
filterSubdomains="${HBLOCK_FILTER_SUBDOMAINS-"false"}"
# Number of times to retry a failed download.
retry="${HBLOCK_RETRY-"0"}"
# Abort if a download error occurs.
continue="${HBLOCK_CONTINUE-"false"}"
# Maximum concurrency for parallel downloads.
parallel="${HBLOCK_PARALLEL-"4"}"
# Colorize the output.
color="${HBLOCK_COLOR-"auto"}"
# Suppress non-error messages.
quiet="${HBLOCK_QUIET-"false"}"
# Parse command line options.
# shellcheck disable=SC2086
{ optParse "${@-}"; _IFS="${IFS?}"; IFS="${SEP:?}"; set -- ${POS-} >/dev/null; IFS="${_IFS?}"; }
# Define terminal colors if the color option is enabled or in auto mode if STDOUT is attached to a TTY and the
# "NO_COLOR" variable is not set (https://no-color.org).
if [ "${color:?}" = 'true' ] || { [ "${color:?}" = 'auto' ] && [ -z "${NO_COLOR+x}" ] && [ -t 1 ]; }; then
COLOR_RESET="$({ exists tput && tput sgr0; } 2>/dev/null || printf '\033[0m')"
COLOR_BRED="$({ exists tput && tput bold && tput setaf 1; } 2>/dev/null || printf '\033[1;31m')"
COLOR_BGREEN="$({ exists tput && tput bold && tput setaf 2; } 2>/dev/null || printf '\033[1;32m')"
COLOR_BYELLOW="$({ exists tput && tput bold && tput setaf 3; } 2>/dev/null || printf '\033[1;33m')"
COLOR_BCYAN="$({ exists tput && tput bold && tput setaf 6; } 2>/dev/null || printf '\033[1;36m')"
fi
# Set "NO_STDOUT" variable if the quiet option is enabled (other methods will honor this variable).
if [ "${quiet:?}" = 'true' ]; then
NO_STDOUT='true'
fi
# Check the header file.
case "${headerFile:?}" in
# If the file value equals "-", use stdin.
'-') headerFile="$(createTemp 'file')"; cat <&0 > "${headerFile:?}" ;;
# If the file value equals "none", use an empty file.
'none') headerFile="$(createTemp 'file')" ;;
# If the file value equals "builtin", use the built-in value.
'builtin') headerFile="$(createTemp 'file')"; printf '%s' "${HBLOCK_HEADER_BUILTIN?}" > "${headerFile:?}" ;;
# If the file does not exist, throw an error.
*) [ -e "${headerFile:?}" ] || { printError "No such file: ${headerFile:?}"; exit 1; } ;;
esac
# Check the footer file.
case "${footerFile:?}" in
# If the file value equals "-", use stdin.
'-') footerFile="$(createTemp 'file')"; cat <&0 > "${footerFile:?}" ;;
# If the file value equals "none", use an empty file.
'none') footerFile="$(createTemp 'file')" ;;
# If the file value equals "builtin", use the built-in value.
'builtin') footerFile="$(createTemp 'file')"; printf '%s' "${HBLOCK_FOOTER_BUILTIN?}" > "${footerFile:?}" ;;
# If the file does not exist, throw an error.
*) [ -e "${footerFile:?}" ] || { printError "No such file: ${footerFile:?}"; exit 1; } ;;
esac
# Check the sources file.
case "${sourcesFile:?}" in
# If the file value equals "-", use stdin.
'-') sourcesFile="$(createTemp 'file')"; cat <&0 > "${sourcesFile:?}" ;;
# If the file value equals "none", use an empty file.
'none') sourcesFile="$(createTemp 'file')" ;;
# If the file value equals "builtin", use the built-in value.
'builtin') sourcesFile="$(createTemp 'file')"; printf '%s' "${HBLOCK_SOURCES_BUILTIN?}" > "${sourcesFile:?}" ;;
# If the file does not exist, throw an error.
*) [ -e "${sourcesFile:?}" ] || { printError "No such file: ${sourcesFile:?}"; exit 1; } ;;
esac
# Check the allowlist file.
case "${allowlistFile:?}" in
# If the file value equals "-", use stdin.
'-') allowlistFile="$(createTemp 'file')"; cat <&0 > "${allowlistFile:?}" ;;
# If the file value equals "none", use an empty file.
'none') allowlistFile="$(createTemp 'file')" ;;
# If the file value equals "builtin", use the built-in value.
'builtin') allowlistFile="$(createTemp 'file')"; printf '%s' "${HBLOCK_ALLOWLIST_BUILTIN?}" > "${allowlistFile:?}" ;;
# If the file does not exist, throw an error.
*) [ -e "${allowlistFile:?}" ] || { printError "No such file: ${allowlistFile:?}"; exit 1; } ;;
esac
# Check the denylist file.
case "${denylistFile:?}" in
# If the file value equals "-", use stdin.
'-') denylistFile="$(createTemp 'file')"; cat <&0 > "${denylistFile:?}" ;;
# If the file value equals "none", use an empty file.
'none') denylistFile="$(createTemp 'file')" ;;
# If the file value equals "builtin", use the built-in value.
'builtin') denylistFile="$(createTemp 'file')"; printf '%s' "${HBLOCK_DENYLIST_BUILTIN?}" > "${denylistFile:?}" ;;
# If the file does not exist, throw an error.
*) [ -e "${denylistFile:?}" ] || { printError "No such file: ${denylistFile:?}"; exit 1; } ;;
esac
# Create an empty blocklist file.
blocklistFile="$(createTemp 'file')"
# If the sources file is not empty, each source is downloaded and appended to the blocklist file.
if [ -s "${sourcesFile:?}" ]; then
printInfo 'Downloading sources'
sourcesDlDir="$(createTemp 'dir')"
sourcesUrlFile="$(createTemp 'file')"
# Read the sources file ignoring comments or empty lines.
removeComments < "${sourcesFile:?}" > "${sourcesUrlFile:?}"
while IFS= read -r url || [ -n "${url?}" ]; do
# Wait if the number of running jobs exceeds the concurrency limit.
if [ "${parallel:?}" -gt '0' ]; then
while [ "${parallel:?}" -le "$(dirCount "${sourcesDlDir:?}"/*.part)" ]; do
# POSIX does not specify the "-n" option, wait for the last PID as fallback.
# shellcheck disable=SC3045
wait -n 2>/dev/null || wait "${!}"
done
fi
# Initialize the download job and send it to the background.
printList "${url:?}"
sourceDlFile="${sourcesDlDir:?}"/"$(rand)"
touch -- "${sourceDlFile:?}.part"
{
attempt=0
maxAttempts=$((retry + 1))
while true; do
if { fetchUrl "${url:?}" && printf '\n'; } > "${sourceDlFile:?}.part"; then
if [ -e "${sourceDlFile:?}.part" ]; then
mv -- "${sourceDlFile:?}.part" "${sourceDlFile:?}" 2>/dev/null
fi
break
else
rm -f -- "${sourceDlFile:?}.part"
if [ "${continue:?}" = 'true' ]; then
printWarn "Cannot obtain source: ${url:?}"
else
attempt=$((attempt + 1))
if [ "${attempt}" -ge "${maxAttempts}" ]; then
rm -f -- "${sourceDlFile:?}.part"
if [ "${continue:?}" = 'true' ]; then
printWarn "Cannot obtain source: ${url:?}"
break
else
printError "Cannot obtain source: ${url:?}"
{ kill -s USR2 "${$}"; exit 1; } 2>/dev/null
fi
else
sleep 1
fi
fi
fi
done
} &
done < "${sourcesUrlFile:?}"
wait
# Append downloaded sources to the blocklist file.
cat -- "${sourcesDlDir:?}"/* >> "${blocklistFile:?}"
rm -rf -- "${sourcesDlDir:?}"
fi
# If the denylist file is not empty, it is appended to the blocklist file.
if [ -s "${denylistFile:?}" ]; then
printInfo 'Applying denylist'
cat -- "${denylistFile:?}" >> "${blocklistFile:?}"
fi
# If the blocklist file is not empty, it is sanitized.
if [ -s "${blocklistFile:?}" ]; then
printInfo 'Sanitizing blocklist'
sanitizeBlocklist "${lenient:?}" < "${blocklistFile:?}" | removeReservedTLDs | sponge "${blocklistFile:?}"
fi
# If the allowlist file is not empty, the entries on it are removed from the blocklist file.
if [ -s "${allowlistFile:?}" ]; then
printInfo 'Applying allowlist'
allowlistPatternFile="$(createTemp 'file')"
# Entries are treated as regexes depending on whether the regex option is enabled.
removeComments < "${allowlistFile:?}" >> "${allowlistPatternFile:?}"
if [ "${regex:?}" = 'true' ]; then
grep -vf "${allowlistPatternFile:?}" -- "${blocklistFile:?}" | sponge "${blocklistFile:?}"
else
grep -Fxvf "${allowlistPatternFile:?}" -- "${blocklistFile:?}" | sponge "${blocklistFile:?}"
fi
rm -f -- "${allowlistPatternFile:?}"
fi
# If the blocklist file is not empty, it is filtered and sorted.
if [ -s "${blocklistFile:?}" ]; then
if [ "${filterSubdomains:?}" = 'true' ]; then
printInfo 'Filtering redundant subdomains'
awkReverseScript="$(cat <<-'EOF'
BEGIN { FS = "." }
{
for (i = NF; i > 0; i--) {
printf("%s%s", $i, (i > 1 ? FS : RS))
}
}
EOF
)"
awkFilterScript="$(cat <<-'EOF'
BEGIN { p = "." }
{
if (index($0, p) != 1) {
print($0); p = $0"."
}
}
EOF
)"
awk "${awkReverseScript:?}" < "${blocklistFile:?}" | sort \
| awk "${awkFilterScript:?}" | awk "${awkReverseScript:?}" \
| sponge "${blocklistFile:?}"
fi
printInfo 'Sorting blocklist'
sort < "${blocklistFile:?}" | uniq | sponge "${blocklistFile:?}"
fi
# Count blocked domains.
blocklistCount="$(wc -l < "${blocklistFile:?}" | awk '{print($1)}')"
# If the blocklist file is not empty, the format template is applied.
if [ -s "${blocklistFile:?}" ]; then
printInfo 'Applying format template'
# The number of domains per line is equal to the value of the wrap option.
if [ "${wrap:?}" -gt '1' ]; then
awkWrapScript='{ORS=(NR%W?FS:RS)}1;END{if(NR%W){printf(RS)}}'
awk -v FS=' ' -v RS='\n' -v W="${wrap:?}" "${awkWrapScript:?}" < "${blocklistFile:?}" \
| sponge "${blocklistFile:?}"
fi
# The following awk script replaces in the template the variables starting with a % sign with their value.
awkTemplateScript="$(cat <<-'EOF'
BEGIN {
Tl = length(T); split(T, Ta, "")
for (i = 1; i <= Tl; i++) {
if (Ta[i] == "%") {
i++; if (Ta[i] == "D") { Vn[++Vl] = "D"; Vp[Vl] = i - 1 }
else if (Ta[i] == "R") { Vn[++Vl] = "R"; Vp[Vl] = i - 1 }
else if (Ta[i] == "%") { Vn[++Vl] = "%"; Vp[Vl] = i - 1 }
}
}
}
{
o = T
for (i = Vl; i > 0 ; i--) {
if (Vn[i] == "D") v = $0
else if (Vn[i] == "R") v = R
else if (Vn[i] == "%") v = "%"
else v = ""
o = substr(o, 1, Vp[i] - 1) v substr(o, Vp[i] + 2)
}
print(o)
}
EOF
)"
awk -v T="${template?}" -v R="${redirection?}" "${awkTemplateScript:?}" < "${blocklistFile:?}" \
| sponge "${blocklistFile:?}"
fi
printOutputFile() {
# Define "C" variable for convenience.
C="${comment?}"
# Append banner to the output file.
if [ -n "${C?}" ]; then
cat <<-EOF
${C?} Generated with hBlock ${HOSTS_VERSION:?} (${HOSTS_REPOSITORY:?})
${C?} Blocked domains: ${blocklistCount:?}
EOF
if [ -z "${SOURCE_DATE_EPOCH+x}" ]; then
cat <<-EOF
${C?} Date: $(date)
EOF
fi
fi
# If the header file is not empty, it is appended to the output file.
if [ -s "${headerFile:?}" ]; then
[ -z "${C?}" ] || printf '\n%s\n' "${C?} BEGIN HEADER"
awk 1 < "${headerFile:?}"
[ -z "${C?}" ] || printf '%s\n' "${C?} END HEADER"
fi
# If the blocklist file is not empty, it is appended to the output file.
if [ -s "${blocklistFile:?}" ]; then
[ -z "${C?}" ] || printf '\n%s\n' "${C?} BEGIN BLOCKLIST"
awk 1 < "${blocklistFile:?}"
[ -z "${C?}" ] || printf '%s\n' "${C?} END BLOCKLIST"
fi
# If the footer file is not empty, it is appended to the output file.
if [ -s "${footerFile:?}" ]; then
[ -z "${C?}" ] || printf '\n%s\n' "${C?} BEGIN FOOTER"
awk 1 < "${footerFile:?}"
[ -z "${C?}" ] || printf '%s\n' "${C?} END FOOTER"
fi
}
# If the file name equals "-", print to stdout.
if [ "${outputFile:?}" = '-' ]; then
printOutputFile
# Try writing the file.
elif touch -- "${outputFile:?}" >/dev/null 2>&1; then
printOutputFile > "${outputFile:?}"
# If writing fails, try with sudo.
elif exists sudo && exists tee; then
printOutputFile | sudo tee -- "${outputFile:?}" >/dev/null
# Throw an error for everything else.
else
printError "Cannot write file: ${outputFile:?}"
exit 1
fi
printInfo "${blocklistCount:?} blocked domains!"
}
main "${@-}"
Reference in a new issue View git blame Copy permalink