diff --git a/scripts/crawler.py b/scripts/crawler.py index 053c388..9bc35a3 100644 --- a/scripts/crawler.py +++ b/scripts/crawler.py @@ -116,7 +116,7 @@ def add_urls(urls): title = html.title.string if html.title is not None else "" if blacklist_file is not None: for row in blacklist_file.itertuples(): - if row[1] in title: + if row[1].lower() in title.lower(): print_colors('[+] Forbidden word found. Rejecting.') forbidden = True if forbidden: