oxeo0/darknet-lantern
1
0
Fork 0
mirror of http://git.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/nihilist/darknet-lantern.git synced 2025-05-16 20:26:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

switch to IsUrlValid

Browse source
This commit is contained in:
cynthia 2025-04-03 21:53:37 +00:00
parent 0a0233a8a2
commit 08e2b5ab20
1 changed files with 2 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

6
scripts/crawler.py
View file

@ -5,7 +5,7 @@ import urllib.parse
import argparse import argparse
import os import os
import re import re
from utils import print_colors from utils import print_colors, IsUrlValid
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog='Lantern crawler', prog='Lantern crawler',
@ -99,9 +99,7 @@ def extract_urls_html(url, text):
print_colors(f'[D] Joined URL: {joined_url}') print_colors(f'[D] Joined URL: {joined_url}')
# Check if the URL is a .onion link or not even a web link # Check if the URL is a .onion link or not even a web link
if jurl_parsed.scheme != 'http': if not IsUrlValid(joined_url):
continue
if not jurl_parsed.hostname.endswith('.onion'):
continue continue
print_colors(f'[+] Found url: {joined_url}') print_colors(f'[+] Found url: {joined_url}')