From 9493fe420a4abc20fdf8b365eff5d19a80798d00 Mon Sep 17 00:00:00 2001 From: SovereigntyIsNotFreedom Date: Fri, 24 Jan 2025 15:09:22 +0300 Subject: [PATCH 1/2] Added highlighting and other files for maintainablility --- .gitignore | 1 + requirements.txt | 12 + scripts/__pycache__/utils.cpython-313.pyc | Bin 0 -> 6703 bytes scripts/lantern.py | 609 ++++++++-------------- scripts/utils.py | 259 +++++++++ 5 files changed, 476 insertions(+), 405 deletions(-) create mode 100644 .gitignore create mode 100644 requirements.txt create mode 100644 scripts/__pycache__/utils.cpython-313.pyc create mode 100644 scripts/utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..191381e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.git \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..af698b3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +certifi==2024.12.14 +charset-normalizer==3.4.1 +idna==3.10 +numpy==2.2.2 +pandas==2.2.3 +python-dateutil==2.9.0.post0 +python-socks==2.6.1 +pytz==2024.2 +requests==2.32.3 +six==1.17.0 +tzdata==2025.1 +urllib3==2.3.0 diff --git a/scripts/__pycache__/utils.cpython-313.pyc b/scripts/__pycache__/utils.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e32a362467231811e4994bd77d1c8271895c07b8 GIT binary patch literal 6703 zcmb_gTTC3+89ptwh?HGD$?emJarqpQIz(f|3AYH zYrL-As%JIlocYh0bNl}9JLl}3+FBQZaO$h8@l*#Pf5De>ux)4V{01^Nh(sjjSwe}# zQbyt^E7>R~*{Mx(P`l*ZL#RXI&pV}>^ZcMqa_u2su+bVxuqL^v@aAe#?qEv~tZ8b1`OnIWTd+!@+asuMok~Fz{o;tjg>q z6JRmJIM&z|UWTQUY{$9r^5RyMt8A2&>_e{;B12zsN8GuuvNh~;S8x&;;8VHBiNvei zFc){4=?qgb3Vzo|@H%dQey{z(ZOxF+Ko2uas3z0n8ktpMuW4dpMvRW16%(2$$I=UO zGPx*HMUKtNQ%Oac{x3+JE)Rgpz1x|jI%eq`8$xgK_2GAQF3^j3n~+3 z#;f?{0!zsJV3dLb#82{1T)+^2~h-G z!G2~=jw`w?HLs|8&1l(WbeopQC?Pwohk81jn4^Bo*k4af;{whqiTG?aBpVI{bdeday}w@6syFpHLMCt#j1Xn3w(2O4pqxt1A#ixP`G5IV<>Gv0 zk5-#CL*NcF>Ng*EJ#5^F8Pww+P|xDouK878wMU!ZBYwMUe%45&N12~hyF5Q&yRVE6 zu)PT|wN0Q(8bK1GHV+xF&=d(MfnBu?+v0XJtxy7Y=7n8KFrAu{6KdDIOy#+du_K9> zkb`1M1^SC-u_&w4utTBri4nS>h)PmZ#4#D-uq*S}#fjmbSLL2eU(fOIg@aw^!^XFv zr#f_|H{--T3bwEIFfN1JI;_8a_KBpbZ0CzHxGiO+qE=o5W1Tx zbE!*;wlFi3xU9Qi%b{uQ^#llsU7JrP(t6FzLNYlgr(?4^msC^=?g&~lPZMeyBnX`v zwnaIOdyeifC!@R7No`>YUJ|mZ8S1oKj@)Oo=WzO_8Vx`zL-1&y1&0GQ^7YzVH$Z(3 zeEY!d)?>xGw@RnlUJY2h1P_Qg3tl7)cDxV zSFXR3(^fC$hYJn+m&ewNZJGc>O%wD>A<@%NUNHN$|41{hC7Xv*T z?fdhWet6~kSKb{db{;FXAJ3kJ)>`&`7~GRPviha`crnRFTNZ2 z!B>jyM=JUnSiPKYTmACxt_F(!&Vs*pr>jpkgnE!(SSP+V>&!7(ZFz2uZz=FCD8%;G zPqm}4pkF@~8fC~YyS*5HrVip?`G;MQ{JI5AzwSZPdyH$eoqJF4jPB*$+iQpXJz9;< zWN7ujAakSg#P?98p0ISfVoxR$mO}4XCDzg z?iQv|-#Ab=OPgVe+hM0YLnXMlBjn5snB2zWB^=X`%qEqmQt4R{PhcV(!9)eP97`%P zg%@CNT?KoGdc!>z4t8we;bJV~FnLTn;4Kp3X(yxAqkP8!A>53L!5q{tH8Jqbrj zH4TcQO-|AtFojryg|rXy8bVUu@?B>%IPIAXJC23;q##-%cYUP3@#fIChi>mZR16F( zJ2yQ3Z2QXWGPfakmbH~*S$XB;X0QeMg~vdCP1siu_H6{)4bt59b!`UstOYv@!OmOU zTClqi>_(3b>{$zR6apQ2320Ra^rJ1>!{b zqUeLWLV3>@Ni|OcpThmq!1!h61f*s@ZVTBLBG6QYP@6|>IGveG3)&1fhKWnvm%IP` z)*Ib_e)Btj``NV%Vev(UPNhJYjbd;DL!i+OOzBppU2qd%7Ij8lz+cmr)`GsnSyZVG$L01>Mk&34?tAfl}r<>6XdhT0bALXOO_ZKC3lTf%uFNCs+c7bkER z?FJUMiMY{;$HY1LvIuuWssUSFJLP}~?xFN4h$3TKcu{@Fu|v`d{E&xb5Qi~;9~oi8 zGsas_J+qw@)Lalqk-JN0^c*>aWJUu!#96cM^BA_GmG^`$Y#~38{LtS@9 zI}PiO*Kg~D0~Y}DQ+TvVi136%+`jCgm7&dGvvtrJEY=+c?7el_!Ii~j+dA*Q=DO;7 z!@UD|q$so%gahj+%--zrqR;|a#J+yzm0VzD5_w*xw`*q^?(qN{48YjVe*~HTj^j}h zAkQ2#EpKrOp~by;BbQ;&>t8RH{^ zl@NdGtpHa9T;az3G%Q2p9eV|>4MpBzG)JF>DhA9tzrCo78ZMz&-f6vOv=P(k7EOM? z0Hwc&NBcZP75u*48ZOrLE<69&)VBQ0y07V`=ep+)K5=K`e7P717Xn8mE3>g3SyQmdtuUiaizoIl+C?5?gS7CdxdZ1w3XL z;h`<;jA0-Av+HVH`R)K6+ZWkB>^D5nVX-+}YEU&qv3fKRc@P#tW7@ObZ;SqWFbvDwUkp z`Ghv9;D4?6GSzK#1xIM?g7mFQDj9bZ=u`?+~Lm^WQ)INBL_$4HzLk*B0DG^BW9d|J) ph3lC1ysbnaxdmN9QtI--htMS?xzl$s`FoeZ^kfGNJ; bool: - """ - Checks if the banner.png file has the correct dimensions (240x60) - """ - #print('[+] checking image size') - try: - im = Image.open(path) - except Exception as e: - return False - #im = Image.open("favicon.png") - width, height = im.size - #print('width =',width, 'height=',height) - if width != 240 or height != 60: - #print('[-] Banner doesnt have the correct size (240x60)') - return False - else: - #print('[+] Banner has the correct size (240x60)') - return True - - -def IsOnionValid(url: str)-> bool: - """ - Checks if the domain(param) is a valid onion domain and return True else False. - """ - # check if the characters are only [a-zA-Z0-9.] with maximum 128 chars max? - # check that it is only url.onion or subdomain.url.onion, - # if OK return True - #if not : return False - try: - pattern = re.compile("^[A-Za-z0-9.]+(\.onion)?$") - url = url.strip().removesuffix('/') - if url.startswith('http://'): - #print('URL starts with http') - # Removes the http:// - domain = url.split('/')[2] - if pattern.fullmatch(domain) is not None: - if len(domain.split('.')) > 3: - n_subdomians = len(domain.split('.')) - # Checks if there is more than 1 subdomain. "subdomain.url.onion" only - #print(f"This domain have more than one subdomain. There are {n_subdomians} subdomains") - return False - else: - if len(domain) < 62: - #print("Domain length is less than 62.") - return False - return True - elif pattern.fullmatch(domain) is None: - #print("Domain contains invalid character.") - #print(domain) - return False - else: - #print("Domain not valid") - return False - else: - #TODO : edit the url to make sure it has http:// at the beginning, in case if it's missing? (problem is that it only returns true or false) - #print("URL doesn't start http") - if pattern.fullmatch(url) is not None: - if len(url.split('.')) > 3: - n_subdomians = len(url.split('.')) - # Checks if there is more than 1 subdomain. "subdomain.url.onion" only - #print(f"This domain have more than one subdomain. There are {n_subdomians - 1} subdomains") - return False - else: - if len(url) < 62: - #print("Domain length is less than 62.") - return False - return True - elif pattern.fullmatch(url) is None: - #print("Domain contains invalid character.") - #print(url) - return False - else: - #print("Domain not valid") - return False - except Exception as e: - print(f"Error: {e}") - -def IsUrlValid(url:str)->bool: - """ - Check if url is valid both dark net end clearnet. - """ - # check if the characters are only [a-zA-Z0-9.:/] with maximum 128 chars max? - # check that it is only http(s)://wordA.wordB or http(s)://WordC.WordB.WordC, (onion or not), clearnet is fine too (double check if those are fine!) - # if OK return True - #if not : return False - pattern = re.compile("^[A-Za-z0-9:/.-]+$") - url = str(url) - if len(url) < 4: - #print("Status: Got more than one character or nothing.") - return False - if url.endswith('.onion'): - return IsOnionValid(url) - else: - if not url.__contains__('.'): - #print("No (DOT) in clearnet url") - return False - if pattern.fullmatch(url) is None: - #print('Url contains invalid chars') - return False - return True - -def IsStatusValid(status: str)-> bool: - """ - Checks if status contains only [v,x,❌,✔️]. Verbose only if False is returned - """ - pattern = ['y','n','✔️','❌','','nan'] - status = str(status) - status.strip() - #print('[+] STATUS = ',status.splitlines()) - if len(status) > 4: - #print("Status: Got more than one character or nothing.") - return False - elif (status not in pattern): - #print("Status: Got an invalid character it must be either y, n, ✔️, or ❌ ") - return False - - return True - - -def IsScoreValid(score:str)->bool: - """ - Check the Score is only "^[0-9.,]+$" with 8 max chars. - """ - # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max - #(careful with the ' and , make sure you test if it fucks the csv up or else) - # if OK return True - #if not : return False - pattern = re.compile("^[0-9.,]+$") - score = str(score) - score.strip() - #pattern = ['','nan'] - if score in ['','nan']: - #Score can be empty when initially added - return True - if pattern.fullmatch(score) is None: - # empty description is fine as it's optional - return False - elif len(score) > 8: - #print("score is greater than 8 chars") - return False - # empty score is fine - return True - - -def IsDescriptionValid(desc:str)->bool: - """ - Check the categories are only [a-zA-Z0-9.' ] with 256 max chars. - """ - # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max - #(careful with the ' and , make sure you test if it fucks the csv up or else) - # if OK return True - #if not : return False - if desc == "": - # empty description is fine as it's optional - return True - pattern = re.compile("^[A-Za-z0-9-.,' \"]+$") - desc = str(desc) - desc.strip() - if pattern.fullmatch(desc) is None: - return False - if desc == "DEFAULT": - return False - elif len(desc) > 256: - #print("desc is greater than 256 chars") - return False - return True - -def IsCategoryValid(categories: list)-> bool: - """ - Check the categories are only [a-zA-Z0-9 ] with 64 max chars. - """ - # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max - #(careful with the ' and , make sure you test if it fucks the csv up or else) - # if OK return True - #if not : return False - pattern = re.compile("^[A-Za-z0-9 ]+$") - for category in categories: - category.strip() - if pattern.fullmatch(category) is None: - #print('Got an empty list or invalid chars') - return False - elif len(category) > 64: - #print('Category is too long') - return False - else: - return True - -def IsNameValid(name: str)->bool: - """ - Check the parameter name only contains [a-zA-Z0-9 ] and is 64 chars long. - """ - # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max - #(careful with the ' and , make sure you test if it fucks the csv up or else) - # if OK return True - #if not : return False - pattern = re.compile("^[A-Za-z0-9 ]+$") - name = name.strip() - if (pattern.fullmatch(name) is None): - #print("Got an invalid character or nothing") - return False - elif len(name) > 64: - #print(f'Got a name length greater than 64. {len(name)}') - return False - return True - - - - - if __name__ == '__main__': main() diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 0000000..5debf55 --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,259 @@ +import re + +PURPLE = '\033[35;40m' +BOLD_PURPLE = '\033[35;40;1m' +RED = '\033[31;40m' +BOLD_RED = '\033[31;40;1m' +RESET = '\033[m' + + +#### Checking Functions to validate that links are legit #### + +def CheckUrl(url): + """ + Checks if URL is actually reachable via Tor + """ + proxies = { + 'http': 'socks5h://127.0.0.1:9050', + 'https': 'socks5h://127.0.0.1:9050' + } + try: + status = requests.get(url,proxies=proxies, timeout=5).status_code + #print('[+]',url,status) + if status != 502: + #print(url,"✔️") + return True + else: + #print(url,"❌") + return False + except requests.ConnectionError as e: + #print(url,"❌") + return False + except requests.exceptions.ReadTimeout as e: + #print(url,"❌") + return False + +#### PROTECTIONS AGAINST MALICIOUS CSV INPUTS #### + +def IsBannerValid(path: str) -> bool: + """ + Checks if the banner.png file has the correct dimensions (240x60) + """ + #print('[+] checking image size') + try: + im = Image.open(path) + except Exception as e: + return False + #im = Image.open("favicon.png") + width, height = im.size + #print('width =',width, 'height=',height) + if width != 240 or height != 60: + #print('[-] Banner doesnt have the correct size (240x60)') + return False + else: + #print('[+] Banner has the correct size (240x60)') + return True + + +def IsOnionValid(url: str)-> bool: + """ + Checks if the domain(param) is a valid onion domain and return True else False. + """ + # check if the characters are only [a-zA-Z0-9.] with maximum 128 chars max? + # check that it is only url.onion or subdomain.url.onion, + # if OK return True + #if not : return False + try: + pattern = re.compile("^[A-Za-z0-9.]+(\.onion)?$") + url = url.strip().removesuffix('/') + if url.startswith('http://'): + #print('URL starts with http') + # Removes the http:// + domain = url.split('/')[2] + if pattern.fullmatch(domain) is not None: + if len(domain.split('.')) > 3: + n_subdomians = len(domain.split('.')) + # Checks if there is more than 1 subdomain. "subdomain.url.onion" only + #print(f"This domain have more than one subdomain. There are {n_subdomians} subdomains") + return False + else: + if len(domain) < 62: + #print("Domain length is less than 62.") + return False + return True + elif pattern.fullmatch(domain) is None: + #print("Domain contains invalid character.") + #print(domain) + return False + else: + #print("Domain not valid") + return False + else: + #TODO : edit the url to make sure it has http:// at the beginning, in case if it's missing? (problem is that it only returns true or false) + #print("URL doesn't start http") + if pattern.fullmatch(url) is not None: + if len(url.split('.')) > 3: + n_subdomians = len(url.split('.')) + # Checks if there is more than 1 subdomain. "subdomain.url.onion" only + #print(f"This domain have more than one subdomain. There are {n_subdomians - 1} subdomains") + return False + else: + if len(url) < 62: + #print("Domain length is less than 62.") + return False + return True + elif pattern.fullmatch(url) is None: + #print("Domain contains invalid character.") + #print(url) + return False + else: + #print("Domain not valid") + return False + except Exception as e: + print(f"Error: {e}") + +def IsUrlValid(url:str)->bool: + """ + Check if url is valid both dark net end clearnet. + """ + # check if the characters are only [a-zA-Z0-9.:/] with maximum 128 chars max? + # check that it is only http(s)://wordA.wordB or http(s)://WordC.WordB.WordC, (onion or not), clearnet is fine too (double check if those are fine!) + # if OK return True + #if not : return False + pattern = re.compile("^[A-Za-z0-9:/.-]+$") + url = str(url) + if len(url) < 4: + #print("Status: Got more than one character or nothing.") + return False + if url.endswith('.onion'): + return IsOnionValid(url) + else: + if not url.__contains__('.'): + #print("No (DOT) in clearnet url") + return False + if pattern.fullmatch(url) is None: + #print('Url contains invalid chars') + return False + return True + +def IsStatusValid(status: str)-> bool: + """ + Checks if status contains only [v,x,❌,✔️]. Verbose only if False is returned + """ + pattern = ['y','n','✔️','❌','','nan'] + status = str(status) + status.strip() + #print('[+] STATUS = ',status.splitlines()) + if len(status) > 4: + #print("Status: Got more than one character or nothing.") + return False + elif (status not in pattern): + #print("Status: Got an invalid character it must be either y, n, ✔️, or ❌ ") + return False + + return True + + +def IsScoreValid(score:str)->bool: + """ + Check the Score is only "^[0-9.,]+$" with 8 max chars. + """ + # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max + #(careful with the ' and , make sure you test if it fucks the csv up or else) + # if OK return True + #if not : return False + pattern = re.compile("^[0-9.,]+$") + score = str(score) + score.strip() + #pattern = ['','nan'] + if score in ['','nan']: + #Score can be empty when initially added + return True + if pattern.fullmatch(score) is None: + # empty description is fine as it's optional + return False + elif len(score) > 8: + #print("score is greater than 8 chars") + return False + # empty score is fine + return True + + +def IsDescriptionValid(desc:str)->bool: + """ + Check the categories are only [a-zA-Z0-9.' ] with 256 max chars. + """ + # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max + #(careful with the ' and , make sure you test if it fucks the csv up or else) + # if OK return True + #if not : return False + if desc == "": + # empty description is fine as it's optional + return True + pattern = re.compile("^[A-Za-z0-9-.,' \"]+$") + desc = str(desc) + desc.strip() + if pattern.fullmatch(desc) is None: + return False + if desc == "DEFAULT": + return False + elif len(desc) > 256: + #print("desc is greater than 256 chars") + return False + return True + +def IsCategoryValid(categories: list)-> bool: + """ + Check the categories are only [a-zA-Z0-9 ] with 64 max chars. + """ + # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max + #(careful with the ' and , make sure you test if it fucks the csv up or else) + # if OK return True + #if not : return False + pattern = re.compile("^[A-Za-z0-9 ]+$") + for category in categories: + category.strip() + if pattern.fullmatch(category) is None: + #print('Got an empty list or invalid chars') + return False + elif len(category) > 64: + #print('Category is too long') + return False + else: + return True + +def IsNameValid(name: str)->bool: + """ + Check the parameter name only contains [a-zA-Z0-9 ] and is 64 chars long. + """ + # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max + #(careful with the ' and , make sure you test if it fucks the csv up or else) + # if OK return True + #if not : return False + pattern = re.compile("^[A-Za-z0-9 ]+$") + name = name.strip() + if (pattern.fullmatch(name) is None): + #print("Got an invalid character or nothing") + return False + elif len(name) > 64: + #print(f'Got a name length greater than 64. {len(name)}') + return False + return True + + + +def print_colors(s:str, bold=False, is_error = False, default=False): + """ + Helper function to print with colors + """ + if is_error: + print(f"{RED}{s}{RESET}") + elif bold: + print(f"{BOLD_PURPLE}{s}{RESET}") + elif is_error and bold: + print(f"{BOLD_RED}{s}{RESET}") + elif default: + print(f'{s}') + else: + print(f"{PURPLE}{s}{RESET}") + \ No newline at end of file From e4a46e1c5c5a24f3d7868ee08f48e4cb72fa450e Mon Sep 17 00:00:00 2001 From: SovereigntyIsNotFreedom Date: Fri, 24 Jan 2025 15:12:23 +0300 Subject: [PATCH 2/2] Added highlighting and other files for maintainablility --- scripts/lantern.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/lantern.py b/scripts/lantern.py index d5b92b2..3358458 100644 --- a/scripts/lantern.py +++ b/scripts/lantern.py @@ -4,7 +4,6 @@ import urllib import socks, socket, glob from utils import * -#apt install python3-pandas python3-requests python3-socks def main(): proxies = { 'http': 'socks5h://127.0.0.1:9050',