From 08e2b5ab2091bd4a4f23f223b8979657709ec90d Mon Sep 17 00:00:00 2001
From: cynthia <cynthia@noreply.localhost>
Date: Thu, 3 Apr 2025 21:53:37 +0000
Subject: [PATCH] switch to IsUrlValid

---
 scripts/crawler.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/crawler.py b/scripts/crawler.py
index 0724324..fcdf416 100644
--- a/scripts/crawler.py
+++ b/scripts/crawler.py
@@ -5,7 +5,7 @@ import urllib.parse
 import argparse
 import os
 import re
-from utils import print_colors
+from utils import print_colors, IsUrlValid
 
 parser = argparse.ArgumentParser(
     prog='Lantern crawler',
@@ -99,9 +99,7 @@ def extract_urls_html(url, text):
 
         print_colors(f'[D] Joined URL: {joined_url}')
         # Check if the URL is a .onion link or not even a web link
-        if jurl_parsed.scheme != 'http':
-            continue
-        if not jurl_parsed.hostname.endswith('.onion'):
+        if not IsUrlValid(joined_url):
             continue
 
         print_colors(f'[+] Found url: {joined_url}')