diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..191381e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.git \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..af698b3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +certifi==2024.12.14 +charset-normalizer==3.4.1 +idna==3.10 +numpy==2.2.2 +pandas==2.2.3 +python-dateutil==2.9.0.post0 +python-socks==2.6.1 +pytz==2024.2 +requests==2.32.3 +six==1.17.0 +tzdata==2025.1 +urllib3==2.3.0 diff --git a/scripts/__pycache__/utils.cpython-313.pyc b/scripts/__pycache__/utils.cpython-313.pyc new file mode 100644 index 0000000..e32a362 Binary files /dev/null and b/scripts/__pycache__/utils.cpython-313.pyc differ diff --git a/scripts/lantern.py b/scripts/lantern.py index d71f178..3358458 100644 --- a/scripts/lantern.py +++ b/scripts/lantern.py @@ -2,7 +2,8 @@ import os, pwd, re, pandas as pd, requests, shutil from PIL import Image import urllib import socks, socket, glob -#apt install python3-pandas python3-requests python3-socks +from utils import * + def main(): proxies = { 'http': 'socks5h://127.0.0.1:9050', @@ -11,50 +12,84 @@ def main(): rootpath='/srv/darknet-lantern/' urlpath=pwd.getpwuid(os.getuid()).pw_dir+"/.darknet_participant_url" - #print(urlpath) + #print_colors(urlpath) # check if ~/.darknet_participant_url exists, # if exists, instance= the content of ~/.darknet_participant_url (which is the url: such as lantern.nowherejez...onion) + print_colors(""" +; + ED. + E#Wi G: L. ,; + E###G. j. E#, :EW: ,ft f#i + E#fD#W; .. EW, E#t .GEE##; t#E .E#t GEEEEEEEL + E#t t##L ;W, E##j E#t j#K;E###t t#E i#W, ,;;L#K;;. + E#t .E#K, j##, E###D. E#GK#f E#fE#f t#E L#D. t#E + E#t j##f G###, E#jG#W; E##D. E#t D#G t#E :K#Wfff; t#E + E#t :E#K: :E####, E#t t##f E##Wi E#t f#E. t#E i##WLLLLt t#E + E#t t##L ;W#DG##, E#t :K#E: E#jL#D: E#t t#K: t#E .E#L t#E + E#t .D#W; j###DW##, E#KDDDD###iE#t ,K#jE#t ;#W,t#E f#E: t#E + E#tiW#G. G##i,,G##, E#f,t#Wi,,,E#t jDE#t :K#D#E ,WW; t#E + E#K##i :K#K: L##, E#t ;#W: j#t E#t .E##E .D#; t#E + E##D. ;##D. L##, DWi ,KK: ,; .. G#E tt fE + E#t ,,, .,, fE : + L: , + + + L. ,; L. + i EW: ,ft f#i j. EW: ,ft + LE .. E##; t#E GEEEEEEEL .E#t EW, E##; t#E + L#E ;W, E###t t#E ,;;L#K;;. i#W, E##j E###t t#E + G#W. j##, E#fE#f t#E t#E L#D. E###D. E#fE#f t#E + D#K. G###, E#t D#G t#E t#E :K#Wfff; E#jG#W; E#t D#G t#E + E#K. :E####, E#t f#E. t#E t#E i##WLLLLt E#t t##f E#t f#E. t#E + .E#E. ;W#DG##, E#t t#K: t#E t#E .E#L E#t :K#E: E#t t#K: t#E + .K#E j###DW##, E#t ;#W,t#E t#E f#E: E#KDDDD###iE#t ;#W,t#E + .K#D G##i,,G##, E#t :K#D#E t#E ,WW; E#f,t#Wi,,,E#t :K#D#E + .W#G :K#K: L##, E#t .E##E t#E .D#; E#t ;#W: E#t .E##E + :W##########Wt ;##D. L##, .. G#E fE tt DWi ,KK: .. G#E + :,,,,,,,,,,,,,.,,, .,, fE : fE + , , + """, bold=True) isitvalid="n" while isitvalid != "y": if os.path.isfile(urlpath): with open(urlpath) as f: instance = f.read().rstrip() # check if the instance URL domain is valid - #print(urlpath,instance) + #print_colors(urlpath,instance) if IsOnionValid(instance): - print("[+] Instance Name:",instance,IsOnionValid(instance)) + print_colors("[+] Instance Name:",instance,IsOnionValid(instance)) isitvalid="y" else: - print('[-] Invalid instance name in ~/.darknet_participant_url:', instance) + print_colors(f'[-] Invalid instance name in ~/.darknet_participant_url: {instance}',is_error=True ) return False else: - print("[+] Instance Path doesn't exist yet") + print_colors("[+] Instance Path doesn't exist yet") # and ask for the instance URL domain instance = input("What is your Instance domain ? (ex: lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion): ") instancepath=rootpath+'www/participants/'+instance # check if the instance URL domain is valid if IsOnionValid(instance): - print("[+] Instance Name: ",instance,IsUrlValid(instance)) + print_colors("[+] Instance Name: ",instance,IsUrlValid(instance)) else: - print('[-] Invalid instance name in ~/.darknet_participant_url:', instance) + print_colors(f'[-] Invalid instance name in ~/.darknet_participant_url: {instance}', is_error=True ) return False # ask the user if the instance URL is valid ? - print() - print(instance) + print_colors() + print_colors(instance) isitvalid=input("Is this your this your instance domain ? (y/n) ") # if yes, then write it into ~/.darknet_participant_url if isitvalid == "y" : - print("OK writing the instance url to ~/.darknet_participants_url") + print_colors("OK writing the instance url to ~/.darknet_participants_url") with open(urlpath, "w") as file: file.write(instance) - print("[+] file written, let's read it") + print_colors("[+] file written, let's read it") f = open(urlpath,"r") - print(f.read()) - print("[+] Initial Setup Completed!") + print_colors(f.read()) + print_colors("[+] Initial Setup Completed!") myinstance = instance instancepath=rootpath+'www/participants/'+instance @@ -81,11 +116,11 @@ def main(): bldf = pd.read_csv(blcsvfile) sedf = pd.read_csv(secsvfile) webpdf = pd.read_csv(webpcsvfile) - print("[+] file exists, your Webring URL is", instance) + print_colors("[+] file exists, your Webring URL is", instance) isitvalid = "y" while True: - print(""" + print_colors(""" @@ -113,7 +148,7 @@ Maintenance: 0) Exit """) option = input("Select Option? (0-11): ") - print(option) + print_colors(option) match option: @@ -127,7 +162,7 @@ Maintenance: case "1": done = False while done == False: - print("\n[+] Add a new Website entry (into unverified.csv)") + print_colors("\n[+] Add a new Website entry (into unverified.csv)") name='' while(IsNameValid(name) is not True): name = input("What is the Website name ? ") @@ -153,7 +188,7 @@ Maintenance: sensi = '✔️' newrow=[instance,category,name,url,sensi,desc,'',''] - print("[+] NEWROW=",newrow) + print_colors("[+] NEWROW=",newrow) # (rest is automatic: status, score, instance is = '' because it is your own instance) # TODO check if the entry doesn't already exist in verified.csv and in unverified.csv # if it doesnt exist, add it into unverified.csv @@ -161,7 +196,7 @@ Maintenance: uvdf.index = uvdf.index + 1 # shifting index uvdf = uvdf.sort_index() # sorting by index uvdf = uvdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories - print("[+] New row added! now writing the csv file:") + print_colors("[+] New row added! now writing the csv file:") uvdf.to_csv(unverifiedcsvfile, index=False) choice=input("\n[+] Want to add another website ? (y/n) ") if choice == "n": @@ -169,30 +204,30 @@ Maintenance: case "2": - print("[+] Trust a Website entry (move an entry from unverified to verified.csv)") + print_colors("[+] Trust a Website entry (move an entry from unverified to verified.csv)") done = False while done == False: vdf = pd.read_csv(verifiedcsvfile) uvdf = pd.read_csv(unverifiedcsvfile) # search for a word - print(uvdf[['Name','URL']]) + print_colors(uvdf[['Name','URL']]) name='' while(IsNameValid(name) is not True): name = input("What is the Website name you want to trust ? (ex: Nowhere)") filter_uvdf = uvdf[uvdf.Name.str.contains(name)] # NOTE and display only the matching entries in unverified.csv in an array format (display it in CLI). - print(filter_uvdf[['Name','URL']]) + print_colors(filter_uvdf[['Name','URL']]) # check if there are no results, dont proceed if there are none! if filter_uvdf.size == 0: - print("ERROR no results, skipping.") + print_colors("ERROR no results, skipping.",is_error=True) else: # Each of the rows has an index, index=-1 while (index not in filter_uvdf.index): # prompt the user to ask for with row they want to move to verified.csv index = int(input("What is the index of the entry that you want to move to verified.csv ? (ex: 3) ")) - # once selected, it must be able to SAVE and print that row: - print(uvdf.iloc[index].values) + # once selected, it must be able to SAVE and print_colors that row: + print_colors(uvdf.iloc[index].values) newrow=uvdf.iloc[index].values @@ -202,39 +237,39 @@ Maintenance: vdf = vdf.sort_index() # sxorting by index vdf = vdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories vdf.to_csv(verifiedcsvfile, index=False) - print("[+] New row added to verified.csv! now writing to the csv") + print_colors("[+] New row added to verified.csv! now writing to the csv") # remove it from unverified.csv uvdf.drop(index, inplace= True) uvdf = uvdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories uvdf.to_csv(unverifiedcsvfile, index=False) - print("[+] Link is now moved to verified.csv!") + print_colors("[+] Link is now moved to verified.csv!") choice=input("\n[+] Want to trust another website ? (y/n) ") if choice == "n": done = True case "3": - print("[+] Untrust a Website entry (move an entry from verified to unverified.csv)") - print(vdf[['Name','URL']]) + print_colors("[+] Untrust a Website entry (move an entry from verified to unverified.csv)") + print_colors(vdf[['Name','URL']]) # search for a word name='' while(IsNameValid(name) is not True): name = input("What is the Website name you want to untrust ? (ex: BreachForums)") filter_vdf = vdf[vdf.Name.str.contains(name)] # and display only the matching entries in unverified.csv in an array format (display it in CLI). - print(filter_vdf[['Name','URL']]) + print_colors(filter_vdf[['Name','URL']]) # check if there are no results, dont proceed if there are none! if filter_vdf.size == 0: - print("ERROR no results, skipping.") + print_colors("ERROR no results, skipping.", is_error=True) else: # Each of the rows has an index, index=-1 while (index not in filter_vdf.index): # prompt the user to ask for with row they want to move to unverified.csv index = int(input("What is the index of the entry that you want to move to unverified.csv ? (ex: 3) ")) - # once selected, it must be able to SAVE and print that row: - print(vdf.iloc[index].values) + # once selected, it must be able to SAVE and print_colors that row: + print_colors(vdf.iloc[index].values) newrow=vdf.iloc[index].values @@ -243,13 +278,13 @@ Maintenance: uvdf.index = uvdf.index + 1 # shifting index uvdf = uvdf.sort_index() # sorting by index uvdf.to_csv(unverifiedcsvfile, index=False) - print("[+] New row added to unverified.csv!") + print_colors("[+] New row added to unverified.csv!") # remove it from verified.csv vdf.drop(index, inplace= True) vdf.to_csv(verifiedcsvfile, index=False) - print("[+] Link is now moved to unverified.csv!") + print_colors("[+] Link is now moved to unverified.csv!") ####### MANAGING WEBRING PARTICIPANTS ########### @@ -264,10 +299,10 @@ Maintenance: #check if it works when you have a second webring participant case "4": - print("4) Synchronize new links from existing webring participants, into your unverified.csv file") + print_colors("4) Synchronize new links from existing webring participants, into your unverified.csv file") # iterate through each existing directories in www/participants/* to get each webring participant participantsdir=rootpath+'www/participants/' - #print(os.listdir(participantsdir)) + #print_colors(os.listdir(participantsdir)) name='' desc='' trusted='' @@ -277,36 +312,36 @@ Maintenance: wdf = pd.read_csv(webringcsvfile) for participant in os.listdir(participantsdir): participantdir=participantsdir+participant - #print(participant) + #print_colors(participant) # NOTE check if the webring participant is yourself, if it is, then skip it if participant != myinstance: # prod: dont use your own intance #if participant == myinstance: # preprod testing only on your own instance #overwrite the existing files in the participant's directory, with their version (download all the csv files from them again) basewurl='http://'+participant+'/participants/'+participant+'/' - print(basewurl) - print('[+] Downloading the files of ',participant, ": ") + print_colors(basewurl) + print_colors('[+] Downloading the files of ',participant, ": ") w_vcsv=basewurl+'verified.csv' w_uvcsv=basewurl+'unverified.csv' - #print(CheckUrl(w_uvcsv)) + #print_colors(CheckUrl(w_uvcsv)) w_blcsv=basewurl+'blacklist.csv' - #print(CheckUrl(w_blcsv)) + #print_colors(CheckUrl(w_blcsv)) w_scsv=basewurl+'sensitive.csv' - #print(CheckUrl(w_scsv)) + #print_colors(CheckUrl(w_scsv)) w_webcsv=basewurl+'webring-participants.csv' - #print(CheckUrl(w_webcsv)) + #print_colors(CheckUrl(w_webcsv)) # verify that their verified.csv csv file exists at basewurl+'verified.csv' if CheckUrl(w_vcsv) is False or CheckUrl(w_uvcsv) is False or CheckUrl(w_blcsv) is False or CheckUrl(w_scsv) is False or CheckUrl(w_webcsv) is False: - print("[-] Webring Participant isn't reachable, skipping") + print_colors("[-] Webring Participant isn't reachable, skipping", is_error=True) #return False #dont do anything if the webring participant isnt reachable. else: #if the webring participant is reachable, proceed - print("[+] Webring Participant is reachable, updating their csv files:") + print_colors("[+] Webring Participant is reachable, updating their csv files:") for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv']: # FOR EACH CSV FILE TO GET: # URL: basewurl / FILE.CSV # PATH: participantdir / FILE.CSV - #print('[+] DOWNLOADING ',basewurl+i) + #print_colors('[+] DOWNLOADING ',basewurl+i) # download the external csv file and save it into the "text" variable: #response = urllib.request.urlopen(basewurl+i) response = requests.get(basewurl+i, proxies=proxies) @@ -314,13 +349,13 @@ Maintenance: #text = data.decode('utf-8') text = response.text # save the text variable into the destination file: - #print('[+] SAVING IT INTO ',participantdir+'/'+i) + #print_colors('[+] SAVING IT INTO ',participantdir+'/'+i) csvfilepath=participantdir+'/'+i with open(csvfilepath, "w") as file: file.write(text) - #print("[+] file written, let's read it") + #print_colors("[+] file written, let's read it") f = open(csvfilepath,"r") - #print(f.read()) + #print_colors(f.read()) # download the banner.png image: @@ -333,71 +368,71 @@ Maintenance: # SANITY CHECK ON THE BANNER PNG IMAGE: if IsBannerValid(bannerpath): - #print('[+] Banner is valid') + #print_colors('[+] Banner is valid') pass else: # if false, overwrite it with the template banner png file - #print('[-] Banner is not valid, replacing it with the default banner') + #print_colors('[-] Banner is not valid, replacing it with the default banner') os.remove(bannerpath) # copy templates/banner.png to bannerpath bannertemplatepath=templatepath+'banner.png' shutil.copyfile(bannertemplatepath, bannerpath) - #print("[+] Webring Participant is valid, adding it if it's not already added.") - #print('[+] PARTICIPANT=',participant) + #print_colors("[+] Webring Participant is valid, adding it if it's not already added.") + #print_colors('[+] PARTICIPANT=',participant) # check if the participant is already listed in webring-participants.csv or not, and add them if not already listed # and display only the matching entries in unverified.csv in an array format (display it in CLI). filter_wdf = wdf[wdf.URL.str.contains(participant)] - #print(filter_wdf[['Name','URL']]) + #print_colors(filter_wdf[['Name','URL']]) # check if there are no results, dont proceed if there are none! if filter_wdf.size == 0: #skip if webring participant is already listed, otherwise proceed newrow=[name,participant,desc,trusted,status,score] - #print("[+] NEWROW=",newrow) + #print_colors("[+] NEWROW=",newrow) wdf.loc[-1] = newrow # adding a row wdf.index = wdf.index + 1 # shifting index wdf = wdf.sort_index() # sorting by index - #print("[+] New row added! now writing the csv file:",webringcsvfile) + #print_colors("[+] New row added! now writing the csv file:",webringcsvfile) wdf.to_csv(webringcsvfile, index=False) else: pass - #print('[+] Webring participant is already listed in your own webring-participants.csv file!') + #print_colors('[+] Webring participant is already listed in your own webring-participants.csv file!') # iterate through the participant's verified.csv and unverified.csv files for w in ['verified.csv','unverified.csv']: csvfilepath=participantdir+'/'+w - print(csvfilepath) + print_colors(csvfilepath) csvdf = pd.read_csv(csvfilepath) - #print(bldf[['blacklisted-words']]) + #print_colors(bldf[['blacklisted-words']]) bldf[['blacklisted-words']].iterrows() rows2delete= [] # it is an empty list at first for i,j in csvdf.iterrows(): - #print("[+] Unverified.csv ROW=",i, uvdf.at[i, 'Instance'], uvdf.at[i, 'Category'], uvdf.at[i, 'Name'], uvdf.at[i, 'URL'], uvdf.at[i, 'Description']) - #print("[+] Unverified.csv ROW=",i, uvdf.iloc[[i]]) + #print_colors("[+] Unverified.csv ROW=",i, uvdf.at[i, 'Instance'], uvdf.at[i, 'Category'], uvdf.at[i, 'Name'], uvdf.at[i, 'URL'], uvdf.at[i, 'Description']) + #print_colors("[+] Unverified.csv ROW=",i, uvdf.iloc[[i]]) #row=uvdf.iloc[[i]] #it displays the index row=csvdf.loc[i,:].values.tolist() - print(row) - #print(i,row) + print_colors(row) + #print_colors(i,row) ################################ SANITY CHECKS #################################### ### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion### - #print("[+] ROW=",i,"ROW CONTENTS=", IsUrlValid(uvdf.at[i, 'Instance']), IsCategoryValid(uvdf.at[i, 'Category']), IsNameValid(uvdf.at[i, 'Name']), IsUrlValid(uvdf.at[i, 'URL']), IsStatusValid(uvdf.at[i, 'Sensitive']), IsDescriptionValid(uvdf.at[i, 'Description']), IsStatusValid(uvdf.at[i, 'Status']), IsScoreValid(uvdf.at[i, 'Score'])) + #print_colors("[+] ROW=",i,"ROW CONTENTS=", IsUrlValid(uvdf.at[i, 'Instance']), IsCategoryValid(uvdf.at[i, 'Category']), IsNameValid(uvdf.at[i, 'Name']), IsUrlValid(uvdf.at[i, 'URL']), IsStatusValid(uvdf.at[i, 'Sensitive']), IsDescriptionValid(uvdf.at[i, 'Description']), IsStatusValid(uvdf.at[i, 'Status']), IsScoreValid(uvdf.at[i, 'Score'])) if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False: #mark the row for deletion as it has invalid inputs if i not in rows2delete: - print("Marking row", i,"for deletion, as it has invalid inputs") + print_colors("Marking row", i,"for deletion, as it has invalid inputs") rows2delete.append(i) #mark the row for deletion if not already done ### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ### for k,l in bldf.iterrows(): - #print("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) + #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) blword=bldf.at[k, 'blacklisted-words'] if any(blword in str(x) for x in row) == True: - #print("found blacklisted word! marking row for deletion") + #print_colors("found blacklisted word! marking row for deletion") if i not in rows2delete: - print("Marking row", i,"for deletion, as it matches with a blacklisted word") + print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word") rows2delete.append(i) #mark the row for deletion if not already done else: if i not in rows2delete: @@ -416,9 +451,9 @@ Maintenance: uvdf.index = uvdf.index + 1 # shifting index uvdf = uvdf.sort_index() # sorting by index uvdf.to_csv(unverifiedcsvfile, index=False) - print("[+] New row added to your own unverified.csv file!") + print_colors("[+] New row added to your own unverified.csv file!") else: - print('[-] Skipping row as it is already added in ',w,row,) + print_colors('[-] Skipping row as it is already added in {w} {row}',is_error=True) @@ -427,18 +462,18 @@ Maintenance: ### SANITY CHECK 3: Mark all the rows that are supposed to be sensitive ### for k,l in sedf.iterrows(): - #print("[+] Sensitive word=",k, sedf.at[k, 'sensitive-words']) + #print_colors("[+] Sensitive word=",k, sedf.at[k, 'sensitive-words']) seword=sedf.at[k, 'sensitive-words'] if any(seword in str(x) for x in row) == True: if csvdf.at[i, 'Sensitive'] != '✔️': - print("Marking row", i,"as sensitive, as it matches with a sensitive word") + print_colors("Marking row", i,"as sensitive, as it matches with a sensitive word") csvdf.at[i, 'Sensitive']='✔️' - print('[-] Rows to delete: ',rows2delete) + print_colors(f'[-] Rows to delete: {rows2delete}', is_error=True) # only delete rows after you've gone through all the unverified.csv OR verified.csv rows' for i in rows2delete: row=csvdf.loc[i,:].values.tolist() - print('[+] REMOVING ROW :',i,row) + print_colors('[+] REMOVING ROW :',i,row) csvdf.drop(i, inplace= True) csvdf.to_csv(csvfilepath, index=False) rows2delete= [] # it is an empty list at first @@ -452,7 +487,7 @@ Maintenance: case "5": - print("[+] Add a new webring participant (and download their files into their directory (without trusting them yet!))") + print_colors("[+] Add a new webring participant (and download their files into their directory (without trusting them yet!))") webring_participant_url = '' while(IsOnionValid(webring_participant_url) is not True): # ask for the url to the other webring participant and check if the (onion only) url is valid or not: @@ -462,30 +497,30 @@ Maintenance: #if not os.path.isdir(participantdir): # to test on your own instance if os.path.isdir(participantdir): # if it does, it means that the webring is ALREADY added - print("[-] Webring Participant is already listed, skipping.") + print_colors("[-] Webring Participant is already listed, skipping.",is_error=True) return False else: # if not, then proceed: - # print the URL to the csv files at http://URL.onion/participants/URL.onion/{verified.csv,unverified.csv,sensitive.csv,blacklist.csv,webring-participants.csv} + # print_colors the URL to the csv files at http://URL.onion/participants/URL.onion/{verified.csv,unverified.csv,sensitive.csv,blacklist.csv,webring-participants.csv} basewurl='http://'+webring_participant_url+'/participants/'+webring_participant_url+'/' - print(basewurl) - print('[+] Checking if all of the required csv files exists for new webring participant ',webring_participant_url, ": ") + print_colors(basewurl) + print_colors('[+] Checking if all of the required csv files exists for new webring participant ',webring_participant_url, ": ") w_vcsv=basewurl+'verified.csv' w_uvcsv=basewurl+'unverified.csv' - #print(CheckUrl(w_uvcsv)) + #print_colors(CheckUrl(w_uvcsv)) w_blcsv=basewurl+'blacklist.csv' - #print(CheckUrl(w_blcsv)) + #print_colors(CheckUrl(w_blcsv)) w_scsv=basewurl+'sensitive.csv' - #print(CheckUrl(w_scsv)) + #print_colors(CheckUrl(w_scsv)) w_webcsv=basewurl+'webring-participants.csv' - #print(CheckUrl(w_webcsv)) + #print_colors(CheckUrl(w_webcsv)) # verify that their verified.csv csv file exists at basewurl+'verified.csv' if CheckUrl(w_vcsv) is False or CheckUrl(w_uvcsv) is False or CheckUrl(w_blcsv) is False or CheckUrl(w_scsv) is False or CheckUrl(w_webcsv) is False: - print("[-] Webring Participant is invalid, exiting.") + print_colors("[-] Webring Participant is invalid, exiting.",is_error=True) return False else: - print("[+] Webring Participant is valid, adding it.") + print_colors("[+] Webring Participant is valid, adding it.") name='' while(IsNameValid(name) is not True): name = input("What is the Webring instance name ? ") @@ -499,11 +534,11 @@ Maintenance: newrow=[name,webring_participant_url,desc,trusted,status,score] webringcsvfile=instancepath+'/'+'webring-participants.csv' wdf = pd.read_csv(webringcsvfile) - #print("[+] NEWROW=",newrow) + #print_colors("[+] NEWROW=",newrow) wdf.loc[-1] = newrow # adding a row wdf.index = wdf.index + 1 # shifting index wdf = wdf.sort_index() # sorting by index - print("[+] New row added! now writing the csv file:",webringcsvfile) + print_colors("[+] New row added! now writing the csv file:",webringcsvfile) wdf.to_csv(webringcsvfile, index=False) # create the directory in www/participants/PARTICIPANTURL/ if it's not there already @@ -516,7 +551,7 @@ Maintenance: # FOR EACH CSV FILE TO GET: # URL: basewurl / FILE.CSV # PATH: participantdir / FILE.CSV - print('[+] DOWNLOADING ',basewurl+i) + print_colors('[+] DOWNLOADING ',basewurl+i) # download the external csv file and save it into the "text" variable: #response = urllib.request.urlopen(basewurl+i) response = requests.get(basewurl+i, proxies=proxies) @@ -524,13 +559,13 @@ Maintenance: #text = data.decode('utf-8') text = response.text # save the text variable into the destination file: - print('[+] SAVING IT INTO ',participantdir+'/'+i) + print_colors('[+] SAVING IT INTO ',participantdir+'/'+i) csvfilepath=participantdir+'/'+i with open(csvfilepath, "w") as file: file.write(text) - print("[+] file written, let's read it") + print_colors("[+] file written, let's read it") f = open(csvfilepath,"r") - print(f.read()) + print_colors(f.read()) # download the banner.png image: @@ -543,10 +578,10 @@ Maintenance: # SANITY CHECK ON THE BANNER PNG IMAGE: if IsBannerValid(bannerpath): - print('[+] Banner is valid') + print_colors('[+] Banner is valid') else: # if false, overwrite it with the template banner png file - print('[-] Banner is not valid, replacing it with the default banner') + print_colors('[-] Banner is not valid, replacing it with the default banner',is_error=True) os.remove(bannerpath) # copy templates/banner.png to bannerpath bannertemplatepath=templatepath+'banner.png' @@ -557,48 +592,48 @@ Maintenance: csvfilepath=participantdir+'/'+w csvdf = pd.read_csv(csvfilepath) - #print(bldf[['blacklisted-words']]) + #print_colors(bldf[['blacklisted-words']]) bldf[['blacklisted-words']].iterrows() rows2delete= [] # it is an empty list at first for i,j in csvdf.iterrows(): - #print("[+] Unverified.csv ROW=",i, uvdf.at[i, 'Instance'], uvdf.at[i, 'Category'], uvdf.at[i, 'Name'], uvdf.at[i, 'URL'], uvdf.at[i, 'Description']) - #print("[+] Unverified.csv ROW=",i, uvdf.iloc[[i]]) + #print_colors("[+] Unverified.csv ROW=",i, uvdf.at[i, 'Instance'], uvdf.at[i, 'Category'], uvdf.at[i, 'Name'], uvdf.at[i, 'URL'], uvdf.at[i, 'Description']) + #print_colors("[+] Unverified.csv ROW=",i, uvdf.iloc[[i]]) #row=uvdf.iloc[[i]] #it displays the index row=csvdf.loc[i,:].values.tolist() - #print(i,row) + #print_colors(i,row) ### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion### - #print("[+] ROW=",i,"ROW CONTENTS=", IsUrlValid(csvdf.at[i, 'Instance']), IsCategoryValid(csvdf.at[i, 'Category']), IsNameValid(csvdf.at[i, 'Name']), IsUrlValid(csvdf.at[i, 'URL']), IsStatusValid(csvdf.at[i, 'Sensitive']), IsDescriptionValid(csvdf.at[i, 'Description']), IsStatusValid(csvdf.at[i, 'Status']), IsScoreValid(csvdf.at[i, 'Score'])) + #print_colors("[+] ROW=",i,"ROW CONTENTS=", IsUrlValid(csvdf.at[i, 'Instance']), IsCategoryValid(csvdf.at[i, 'Category']), IsNameValid(csvdf.at[i, 'Name']), IsUrlValid(csvdf.at[i, 'URL']), IsStatusValid(csvdf.at[i, 'Sensitive']), IsDescriptionValid(csvdf.at[i, 'Description']), IsStatusValid(csvdf.at[i, 'Status']), IsScoreValid(csvdf.at[i, 'Score'])) if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False: #mark the row for deletion as it has invalid inputs if i not in rows2delete: - print("Marking row", i,"for deletion, as it has invalid inputs") + print_colors("Marking row", i,"for deletion, as it has invalid inputs") rows2delete.append(i) #mark the row for deletion if not already done ### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ### for k,l in bldf.iterrows(): - #print("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) + #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) blword=bldf.at[k, 'blacklisted-words'] if any(blword in str(x) for x in row) == True: - #print("found blacklisted word! marking row for deletion") + #print_colors("found blacklisted word! marking row for deletion") if i not in rows2delete: - print("Marking row", i,"for deletion, as it matches with a blacklisted word") + print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word") rows2delete.append(i) #mark the row for deletion if not already done ### SANITY CHECK 3: Mark all the rows that are supposed to be sensitive ### for k,l in sedf.iterrows(): - #print("[+] Sensitive word=",k, sedf.at[k, 'sensitive-words']) + #print_colors("[+] Sensitive word=",k, sedf.at[k, 'sensitive-words']) seword=sedf.at[k, 'sensitive-words'] if any(seword in str(x) for x in row) == True: if csvdf.at[i, 'Sensitive'] != '✔️': - print("Marking row", i,"as sensitive, as it matches with a sensitive word") + print_colors("Marking row", i,"as sensitive, as it matches with a sensitive word") csvdf.at[i, 'Sensitive']='✔️' - print('[-] Rows to delete: ',rows2delete) + print_colors(f'[-] Rows to delete: {rows2delete}',is_error=True) for i in rows2delete: row=csvdf.loc[i,:].values.tolist() - print('[+] REMOVING ROW :',i,row) + print_colors('[+] REMOVING ROW :',i,row) csvdf.drop(i, inplace= True) csvdf.to_csv(csvfilepath, index=False) ############################################## @@ -610,11 +645,11 @@ Maintenance: case "6": - print("[+] Trust/UnTrust/Blacklist a webring participant (Potentially dangerous)") + print_colors("[+] Trust/UnTrust/Blacklist a webring participant (Potentially dangerous)") webringcsvfile=instancepath+'/'+'webring-participants.csv' wdf = pd.read_csv(webringcsvfile) # list each webring participant in your webring-participants.csv file - print(wdf[["URL","Trusted"]]) + print_colors(wdf[["URL","Trusted"]]) # ask the user to pick an index index="" while (index not in wdf.index): @@ -630,32 +665,32 @@ Maintenance: choice2=input("You're about to trust another peer, this means that you're going to automatically trust all of the links they have in their verified.csv file! If this is a malicious peer, you're about to potentially going to automatically trust malicious links, it is potentially risky! Do you want to continue ? (y/n)") if choice2 == "y": # if user wants to proceed, mark the "trusted" column as V - print("[+] Trusting webring participant", wdf.at[index,"URL"]) + print_colors("[+] Trusting webring participant", wdf.at[index,"URL"]) wdf.at[index,"Trusted"]='✔️' wdf.to_csv(webringcsvfile, index=False) else: - print("[-] not trusting webring participant, skipping.") + print_colors("[-] not trusting webring participant, skipping.", is_error=True) if choice == "2": - print("[+] UnTrusting webring participant", wdf.at[index,"URL"]) + print_colors("[+] UnTrusting webring participant", wdf.at[index,"URL"]) wdf.at[index,"Trusted"]='' wdf.to_csv(webringcsvfile, index=False) # untrust the webring participant # if 2: mark the "trusted" column as empty if choice == "3": - print("[+] Blacklisting webring participant", wdf.at[index,"URL"]) + print_colors("[+] Blacklisting webring participant", wdf.at[index,"URL"]) # blacklist the webring participant # add it's URL to your own blacklist.csv instance2blacklist=wdf.at[index,"URL"] newrow=[instance2blacklist] - print("[+] NEWROW=",newrow) + print_colors("[+] NEWROW=",newrow) # (rest is automatic: status, score, instance is = '' because it is your own instance) # check if the entry doesn't already exist in verified.csv and in unverified.csv # if it doesnt exist, add it into unverified.csv bldf.loc[-1] = newrow # adding a row bldf.index = bldf.index + 1 # shifting index bldf = bldf.sort_index() # sorting by index - print("[+] New row added! now writing the csv file:") + print_colors("[+] New row added! now writing the csv file:") bldf.to_csv(blcsvfile, index=False) @@ -666,64 +701,64 @@ Maintenance: for i,j in vdf.iterrows(): row=vdf.loc[i,:].values.tolist() for k,l in bldf.iterrows(): - #print("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) + #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) blword=bldf.at[k, 'blacklisted-words'] if any(blword in str(x) for x in row) == True: - #print("found blacklisted word! marking row for deletion") + #print_colors("found blacklisted word! marking row for deletion") if i not in rows2delete: - print("Marking row", i,"for deletion, as it matches with a blacklisted word") + print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word") rows2delete.append(i) #mark the row for deletion if not already done for i in rows2delete: row=vdf.loc[i,:].values.tolist() - print('[+] REMOVING ROW :',i,row) + print_colors('[+] REMOVING ROW :',i,row) vdf.drop(i, inplace= True) vdf.to_csv(verifiedcsvfile, index=False) - print(vdf) + print_colors(vdf) rows2delete= [] # it is an empty list at first #uvdf rows2delete= [] # it is an empty list at first for i,j in uvdf.iterrows(): row=uvdf.loc[i,:].values.tolist() for k,l in bldf.iterrows(): - #print("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) + #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) blword=bldf.at[k, 'blacklisted-words'] if any(blword in str(x) for x in row) == True: - #print("found blacklisted word! marking row for deletion") + #print_colors("found blacklisted word! marking row for deletion") if i not in rows2delete: - print("Marking row", i,"for deletion, as it matches with a blacklisted word") + print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word") rows2delete.append(i) #mark the row for deletion if not already done for i in rows2delete: row=uvdf.loc[i,:].values.tolist() - print('[+] REMOVING ROW :',i,row) + print_colors('[+] REMOVING ROW :',i,row) uvdf.drop(i, inplace= True) uvdf.to_csv(unverifiedcsvfile, index=False) - print(uvdf) + print_colors(uvdf) rows2delete= [] # it is an empty list at first # find all rows that match with the instance name in wdf aswell to remove them for i,j in wdf.iterrows(): row=wdf.loc[i,:].values.tolist() for k,l in bldf.iterrows(): - #print("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) + #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) blword=bldf.at[k, 'blacklisted-words'] if any(blword in str(x) for x in row) == True: - #print("found blacklisted word! marking row for deletion") + #print_colors("found blacklisted word! marking row for deletion") if i not in rows2delete: - print("Marking row", i,"for deletion, as it matches with a blacklisted word") + print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word") rows2delete.append(i) #mark the row for deletion if not already done for i in rows2delete: row=wdf.loc[i,:].values.tolist() - print('[+] REMOVING ROW :',i,row) + print_colors('[+] REMOVING ROW :',i,row) wdf.drop(i, inplace= True) wdf.to_csv(webringcsvfile, index=False) - print(wdf) + print_colors(wdf) rows2delete= [] # it is an empty list at first # remove the entire directory in www/participants/INSTANCENAME aswell to get rid of it instance2blacklistpath=rootpath+'www/participants/'+instance2blacklist - print("[+] removing the participant's directory at ",instance2blacklistpath) + print_colors("[+] removing the participant's directory at ",instance2blacklistpath) shutil.rmtree(instance2blacklistpath) @@ -741,7 +776,7 @@ Maintenance: ######################################################### case "7": - print("[+] Add/Remove Words/URLs in the sensitive list (ex: drug)") + print_colors("[+] Add/Remove Words/URLs in the sensitive list (ex: drug)") #secsvfile=instancepath+'/sensitive.csv' #fyi #sedf = pd.read_csv(secsvfile) #fyi option="0" @@ -756,25 +791,25 @@ Maintenance: done = True #True to get out of the while loop else: - print("[+] checking if the Word/URL is valid: ") + print_colors("[+] checking if the Word/URL is valid: ") if IsUrlValid(word) or IsOnionValid(word) or IsDescriptionValid(word): - print(IsUrlValid(word), IsOnionValid(word), IsDescriptionValid(word)) - print('[+] Word/URL is valid, adding the word into the sensitive wordlist ') + print_colors(IsUrlValid(word), IsOnionValid(word), IsDescriptionValid(word)) + print_colors('[+] Word/URL is valid, adding the word into the sensitive wordlist ') # add it to the sensitive wordlist newrow=[word] - print("[+] NEWROW=",newrow) + print_colors("[+] NEWROW=",newrow) # (rest is automatic: status, score, instance is = '' because it is your own instance) # check if the entry doesn't already exist in verified.csv and in unverified.csv # if it doesnt exist, add it into unverified.csv sedf.loc[-1] = newrow # adding a row sedf.index = sedf.index + 1 # shifting index sedf = sedf.sort_index() # sorting by index - print("[+] New row added! now writing the csv file: ") + print_colors("[+] New row added! now writing the csv file: ") sedf.to_csv(secsvfile, index=False) if option == "2": - print(sedf) + print_colors(sedf) index="" while (index not in sedf.index) and index != -1: index=int(input("which word do you want to remove? (index 0 to (max index) (write -1 to exit) ")) @@ -784,12 +819,12 @@ Maintenance: else: if (index in sedf.index): #if index exists, remove it - print("[+] removing selected index: ") + print_colors("[+] removing selected index: ") sedf.drop(index, inplace= True) sedf.to_csv(secsvfile, index=False) else: - print('[-] Error, invalid index') + print_colors('[-] Error, invalid index', is_error=True) else: @@ -800,7 +835,7 @@ Maintenance: case "8": - print("[+] Add/Remove words in the blacklist list (ex: porn)") + print_colors("[+] Add/Remove words in the blacklist list (ex: porn)") #blcsvfile=instancepath+'/sensitive.csv' #fyi #bldf = pd.read_csv(secsvfile) #fyi option="0" @@ -815,25 +850,25 @@ Maintenance: done = True #True to get out of the while loop else: - print("[+] Checking if the Word/URL is valid: ") + print_colors("[+] Checking if the Word/URL is valid: ") if IsUrlValid(word) or IsOnionValid(word) or IsDescriptionValid(word): - print(IsUrlValid(word), IsOnionValid(word), IsDescriptionValid(word)) - print('[+] Word/URL is valid, adding the word into the blacklist ') + print_colors(IsUrlValid(word), IsOnionValid(word), IsDescriptionValid(word)) + print_colors('[+] Word/URL is valid, adding the word into the blacklist ') # add it to the sensitive wordlist newrow=[word] - print("[+] NEWROW=",newrow) + print_colors("[+] NEWROW=",newrow) # (rest is automatic: status, score, instance is = '' because it is your own instance) # check if the entry doesn't already exist in verified.csv and in unverified.csv # if it doesnt exist, add it into unverified.csv bldf.loc[-1] = newrow # adding a row bldf.index = bldf.index + 1 # shifting index bldf = bldf.sort_index() # sorting by index - print("[+] New row added! now writing the csv file: ") + print_colors("[+] New row added! now writing the csv file: ") bldf.to_csv(blcsvfile, index=False) if option == "2": - print(bldf) + print_colors(bldf) index="" while (index not in bldf.index) and index != -1: index=int(input("which word do you want to remove? (index 0 to (max index) (write -1 to exit) ")) @@ -843,12 +878,12 @@ Maintenance: else: if (index in bldf.index): #if index exists, remove it - print("[+] removing selected index: ") + print_colors("[+] removing selected index: ") bldf.drop(index, inplace= True) bldf.to_csv(blcsvfile, index=False) else: - print('[-] Error, invalid index') + print_colors('[-] Error, invalid index',is_error=True) else: @@ -861,7 +896,7 @@ Maintenance: # CASE 9 : cleanup all duplicates in unverified + verified.csv, based on the url (check if each url appears more than once, and if they do, remove them + write to csv file) case "9": - print("[+] 9) Cleaning up all duplicates in your own unverified + verified.csv (based on the url)") + print_colors("[+] 9) Cleaning up all duplicates in your own unverified + verified.csv (based on the url)") # ignore it if the index is "indextocheck" and if the index is already listed in rows2delete # else: add the index to "rows2delete" # go drop the rows by their index listed in "rows2delete" @@ -871,17 +906,17 @@ Maintenance: #instancepath=rootpath+'www/participants/'+instance # fyi csvfilepath=instancepath+'/'+w - print(csvfilepath) + print_colors(csvfilepath) csvdf = pd.read_csv(csvfilepath) - print("REMOVING DUPLICATES IN", csvfilepath) + print_colors("REMOVING DUPLICATES IN", csvfilepath) csvdf = csvdf.drop_duplicates(subset=['URL']) csvdf.to_csv(csvfilepath, index=False) - print(csvdf[['URL']]) + print_colors(csvdf[['URL']]) case "10": - print("[+] 10) perform sanity checks on all csv files (to mark them as sensitive / or remove the ones that are blacklisted)") + print_colors("[+] 10) perform sanity checks on all csv files (to mark them as sensitive / or remove the ones that are blacklisted)") participantspath = rootpath+'www/participants/' for participant in os.listdir(participantspath): - print("Participant:",participant) + print_colors("Participant:",participant) participantdir= participantspath+participant a=0 if a == 0: @@ -890,43 +925,43 @@ Maintenance: # iterate through the participant's verified.csv and unverified.csv files for w in ['verified.csv','unverified.csv']: csvfilepath=participantdir+'/'+w - print(csvfilepath) + print_colors(csvfilepath) csvdf = pd.read_csv(csvfilepath) - #print(bldf[['blacklisted-words']]) + #print_colors(bldf[['blacklisted-words']]) bldf[['blacklisted-words']].iterrows() rows2delete= [] # it is an empty list at first for i,j in csvdf.iterrows(): - #print("[+] Unverified.csv ROW=",i, uvdf.at[i, 'Instance'], uvdf.at[i, 'Category'], uvdf.at[i, 'Name'], uvdf.at[i, 'URL'], uvdf.at[i, 'Description']) - #print("[+] Unverified.csv ROW=",i, uvdf.iloc[[i]]) + #print_colors("[+] Unverified.csv ROW=",i, uvdf.at[i, 'Instance'], uvdf.at[i, 'Category'], uvdf.at[i, 'Name'], uvdf.at[i, 'URL'], uvdf.at[i, 'Description']) + #print_colors("[+] Unverified.csv ROW=",i, uvdf.iloc[[i]]) #row=uvdf.iloc[[i]] #it displays the index row=csvdf.loc[i,:].values.tolist() - print(row) - #print(i,row) + print_colors(row) + #print_colors(i,row) ################################ SANITY CHECKS #################################### ### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion### - #print("[+] ROW=",i,"ROW CONTENTS=", IsUrlValid(uvdf.at[i, 'Instance']), IsCategoryValid(uvdf.at[i, 'Category']), IsNameValid(uvdf.at[i, 'Name']), IsUrlValid(uvdf.at[i, 'URL']), IsStatusValid(uvdf.at[i, 'Sensitive']), IsDescriptionValid(uvdf.at[i, 'Description']), IsStatusValid(uvdf.at[i, 'Status']), IsScoreValid(uvdf.at[i, 'Score'])) + #print_colors("[+] ROW=",i,"ROW CONTENTS=", IsUrlValid(uvdf.at[i, 'Instance']), IsCategoryValid(uvdf.at[i, 'Category']), IsNameValid(uvdf.at[i, 'Name']), IsUrlValid(uvdf.at[i, 'URL']), IsStatusValid(uvdf.at[i, 'Sensitive']), IsDescriptionValid(uvdf.at[i, 'Description']), IsStatusValid(uvdf.at[i, 'Status']), IsScoreValid(uvdf.at[i, 'Score'])) if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False: #mark the row for deletion as it has invalid inputs if i not in rows2delete: - print("Marking row", i,"for deletion, as it has invalid inputs") + print_colors("Marking row", i,"for deletion, as it has invalid inputs") rows2delete.append(i) #mark the row for deletion if not already done ### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ### for k,l in bldf.iterrows(): - #print("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) + #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words']) blword=bldf.at[k, 'blacklisted-words'] if any(blword in str(x) for x in row) == True: - #print("found blacklisted word! marking row for deletion") + #print_colors("found blacklisted word! marking row for deletion") if i not in rows2delete: - print("Marking row", i,"for deletion, as it matches with a blacklisted word") + print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word") rows2delete.append(i) #mark the row for deletion if not already done for i in rows2delete: row=csvdf.loc[i,:].values.tolist() - print('[+] REMOVING ROW :',i,row) + print_colors('[+] REMOVING ROW :',i,row) csvdf.drop(i, inplace= True) csvdf.to_csv(csvfilepath, index=False) @@ -938,246 +973,9 @@ Maintenance: # mark the sensitive rows as sensitive # delete the rows that match with blacklisted words case _: - print("[-] Exiting") + print_colors("[-] Exiting",is_error=True) return True -#### Checking Functions to validate that links are legit #### - -def CheckUrl(url): - """ - Checks if URL is actually reachable via Tor - """ - proxies = { - 'http': 'socks5h://127.0.0.1:9050', - 'https': 'socks5h://127.0.0.1:9050' - } - try: - status = requests.get(url,proxies=proxies, timeout=5).status_code - #print('[+]',url,status) - if status != 502: - #print(url,"✔️") - return True - else: - #print(url,"❌") - return False - except requests.ConnectionError as e: - #print(url,"❌") - return False - except requests.exceptions.ReadTimeout as e: - #print(url,"❌") - return False - -#### PROTECTIONS AGAINST MALICIOUS CSV INPUTS #### - -def IsBannerValid(path: str) -> bool: - """ - Checks if the banner.png file has the correct dimensions (240x60) - """ - #print('[+] checking image size') - try: - im = Image.open(path) - except Exception as e: - return False - #im = Image.open("favicon.png") - width, height = im.size - #print('width =',width, 'height=',height) - if width != 240 or height != 60: - #print('[-] Banner doesnt have the correct size (240x60)') - return False - else: - #print('[+] Banner has the correct size (240x60)') - return True - - -def IsOnionValid(url: str)-> bool: - """ - Checks if the domain(param) is a valid onion domain and return True else False. - """ - # check if the characters are only [a-zA-Z0-9.] with maximum 128 chars max? - # check that it is only url.onion or subdomain.url.onion, - # if OK return True - #if not : return False - try: - pattern = re.compile("^[A-Za-z0-9.]+(\.onion)?$") - url = url.strip().removesuffix('/') - if url.startswith('http://'): - #print('URL starts with http') - # Removes the http:// - domain = url.split('/')[2] - if pattern.fullmatch(domain) is not None: - if len(domain.split('.')) > 3: - n_subdomians = len(domain.split('.')) - # Checks if there is more than 1 subdomain. "subdomain.url.onion" only - #print(f"This domain have more than one subdomain. There are {n_subdomians} subdomains") - return False - else: - if len(domain) < 62: - #print("Domain length is less than 62.") - return False - return True - elif pattern.fullmatch(domain) is None: - #print("Domain contains invalid character.") - #print(domain) - return False - else: - #print("Domain not valid") - return False - else: - #TODO : edit the url to make sure it has http:// at the beginning, in case if it's missing? (problem is that it only returns true or false) - #print("URL doesn't start http") - if pattern.fullmatch(url) is not None: - if len(url.split('.')) > 3: - n_subdomians = len(url.split('.')) - # Checks if there is more than 1 subdomain. "subdomain.url.onion" only - #print(f"This domain have more than one subdomain. There are {n_subdomians - 1} subdomains") - return False - else: - if len(url) < 62: - #print("Domain length is less than 62.") - return False - return True - elif pattern.fullmatch(url) is None: - #print("Domain contains invalid character.") - #print(url) - return False - else: - #print("Domain not valid") - return False - except Exception as e: - print(f"Error: {e}") - -def IsUrlValid(url:str)->bool: - """ - Check if url is valid both dark net end clearnet. - """ - # check if the characters are only [a-zA-Z0-9.:/] with maximum 128 chars max? - # check that it is only http(s)://wordA.wordB or http(s)://WordC.WordB.WordC, (onion or not), clearnet is fine too (double check if those are fine!) - # if OK return True - #if not : return False - pattern = re.compile("^[A-Za-z0-9:/.-]+$") - url = str(url) - if len(url) < 4: - #print("Status: Got more than one character or nothing.") - return False - if url.endswith('.onion'): - return IsOnionValid(url) - else: - if not url.__contains__('.'): - #print("No (DOT) in clearnet url") - return False - if pattern.fullmatch(url) is None: - #print('Url contains invalid chars') - return False - return True - -def IsStatusValid(status: str)-> bool: - """ - Checks if status contains only [v,x,❌,✔️]. Verbose only if False is returned - """ - pattern = ['y','n','✔️','❌','','nan'] - status = str(status) - status.strip() - #print('[+] STATUS = ',status.splitlines()) - if len(status) > 4: - #print("Status: Got more than one character or nothing.") - return False - elif (status not in pattern): - #print("Status: Got an invalid character it must be either y, n, ✔️, or ❌ ") - return False - - return True - - -def IsScoreValid(score:str)->bool: - """ - Check the Score is only "^[0-9.,]+$" with 8 max chars. - """ - # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max - #(careful with the ' and , make sure you test if it fucks the csv up or else) - # if OK return True - #if not : return False - pattern = re.compile("^[0-9.,]+$") - score = str(score) - score.strip() - #pattern = ['','nan'] - if score in ['','nan']: - #Score can be empty when initially added - return True - if pattern.fullmatch(score) is None: - # empty description is fine as it's optional - return False - elif len(score) > 8: - #print("score is greater than 8 chars") - return False - # empty score is fine - return True - - -def IsDescriptionValid(desc:str)->bool: - """ - Check the categories are only [a-zA-Z0-9.' ] with 256 max chars. - """ - # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max - #(careful with the ' and , make sure you test if it fucks the csv up or else) - # if OK return True - #if not : return False - if desc == "": - # empty description is fine as it's optional - return True - pattern = re.compile("^[A-Za-z0-9-.,' \"]+$") - desc = str(desc) - desc.strip() - if pattern.fullmatch(desc) is None: - return False - if desc == "DEFAULT": - return False - elif len(desc) > 256: - #print("desc is greater than 256 chars") - return False - return True - -def IsCategoryValid(categories: list)-> bool: - """ - Check the categories are only [a-zA-Z0-9 ] with 64 max chars. - """ - # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max - #(careful with the ' and , make sure you test if it fucks the csv up or else) - # if OK return True - #if not : return False - pattern = re.compile("^[A-Za-z0-9 ]+$") - for category in categories: - category.strip() - if pattern.fullmatch(category) is None: - #print('Got an empty list or invalid chars') - return False - elif len(category) > 64: - #print('Category is too long') - return False - else: - return True - -def IsNameValid(name: str)->bool: - """ - Check the parameter name only contains [a-zA-Z0-9 ] and is 64 chars long. - """ - # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max - #(careful with the ' and , make sure you test if it fucks the csv up or else) - # if OK return True - #if not : return False - pattern = re.compile("^[A-Za-z0-9 ]+$") - name = name.strip() - if (pattern.fullmatch(name) is None): - #print("Got an invalid character or nothing") - return False - elif len(name) > 64: - #print(f'Got a name length greater than 64. {len(name)}') - return False - return True - - - - - if __name__ == '__main__': main() diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 0000000..5debf55 --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,259 @@ +import re + +PURPLE = '\033[35;40m' +BOLD_PURPLE = '\033[35;40;1m' +RED = '\033[31;40m' +BOLD_RED = '\033[31;40;1m' +RESET = '\033[m' + + +#### Checking Functions to validate that links are legit #### + +def CheckUrl(url): + """ + Checks if URL is actually reachable via Tor + """ + proxies = { + 'http': 'socks5h://127.0.0.1:9050', + 'https': 'socks5h://127.0.0.1:9050' + } + try: + status = requests.get(url,proxies=proxies, timeout=5).status_code + #print('[+]',url,status) + if status != 502: + #print(url,"✔️") + return True + else: + #print(url,"❌") + return False + except requests.ConnectionError as e: + #print(url,"❌") + return False + except requests.exceptions.ReadTimeout as e: + #print(url,"❌") + return False + +#### PROTECTIONS AGAINST MALICIOUS CSV INPUTS #### + +def IsBannerValid(path: str) -> bool: + """ + Checks if the banner.png file has the correct dimensions (240x60) + """ + #print('[+] checking image size') + try: + im = Image.open(path) + except Exception as e: + return False + #im = Image.open("favicon.png") + width, height = im.size + #print('width =',width, 'height=',height) + if width != 240 or height != 60: + #print('[-] Banner doesnt have the correct size (240x60)') + return False + else: + #print('[+] Banner has the correct size (240x60)') + return True + + +def IsOnionValid(url: str)-> bool: + """ + Checks if the domain(param) is a valid onion domain and return True else False. + """ + # check if the characters are only [a-zA-Z0-9.] with maximum 128 chars max? + # check that it is only url.onion or subdomain.url.onion, + # if OK return True + #if not : return False + try: + pattern = re.compile("^[A-Za-z0-9.]+(\.onion)?$") + url = url.strip().removesuffix('/') + if url.startswith('http://'): + #print('URL starts with http') + # Removes the http:// + domain = url.split('/')[2] + if pattern.fullmatch(domain) is not None: + if len(domain.split('.')) > 3: + n_subdomians = len(domain.split('.')) + # Checks if there is more than 1 subdomain. "subdomain.url.onion" only + #print(f"This domain have more than one subdomain. There are {n_subdomians} subdomains") + return False + else: + if len(domain) < 62: + #print("Domain length is less than 62.") + return False + return True + elif pattern.fullmatch(domain) is None: + #print("Domain contains invalid character.") + #print(domain) + return False + else: + #print("Domain not valid") + return False + else: + #TODO : edit the url to make sure it has http:// at the beginning, in case if it's missing? (problem is that it only returns true or false) + #print("URL doesn't start http") + if pattern.fullmatch(url) is not None: + if len(url.split('.')) > 3: + n_subdomians = len(url.split('.')) + # Checks if there is more than 1 subdomain. "subdomain.url.onion" only + #print(f"This domain have more than one subdomain. There are {n_subdomians - 1} subdomains") + return False + else: + if len(url) < 62: + #print("Domain length is less than 62.") + return False + return True + elif pattern.fullmatch(url) is None: + #print("Domain contains invalid character.") + #print(url) + return False + else: + #print("Domain not valid") + return False + except Exception as e: + print(f"Error: {e}") + +def IsUrlValid(url:str)->bool: + """ + Check if url is valid both dark net end clearnet. + """ + # check if the characters are only [a-zA-Z0-9.:/] with maximum 128 chars max? + # check that it is only http(s)://wordA.wordB or http(s)://WordC.WordB.WordC, (onion or not), clearnet is fine too (double check if those are fine!) + # if OK return True + #if not : return False + pattern = re.compile("^[A-Za-z0-9:/.-]+$") + url = str(url) + if len(url) < 4: + #print("Status: Got more than one character or nothing.") + return False + if url.endswith('.onion'): + return IsOnionValid(url) + else: + if not url.__contains__('.'): + #print("No (DOT) in clearnet url") + return False + if pattern.fullmatch(url) is None: + #print('Url contains invalid chars') + return False + return True + +def IsStatusValid(status: str)-> bool: + """ + Checks if status contains only [v,x,❌,✔️]. Verbose only if False is returned + """ + pattern = ['y','n','✔️','❌','','nan'] + status = str(status) + status.strip() + #print('[+] STATUS = ',status.splitlines()) + if len(status) > 4: + #print("Status: Got more than one character or nothing.") + return False + elif (status not in pattern): + #print("Status: Got an invalid character it must be either y, n, ✔️, or ❌ ") + return False + + return True + + +def IsScoreValid(score:str)->bool: + """ + Check the Score is only "^[0-9.,]+$" with 8 max chars. + """ + # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max + #(careful with the ' and , make sure you test if it fucks the csv up or else) + # if OK return True + #if not : return False + pattern = re.compile("^[0-9.,]+$") + score = str(score) + score.strip() + #pattern = ['','nan'] + if score in ['','nan']: + #Score can be empty when initially added + return True + if pattern.fullmatch(score) is None: + # empty description is fine as it's optional + return False + elif len(score) > 8: + #print("score is greater than 8 chars") + return False + # empty score is fine + return True + + +def IsDescriptionValid(desc:str)->bool: + """ + Check the categories are only [a-zA-Z0-9.' ] with 256 max chars. + """ + # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max + #(careful with the ' and , make sure you test if it fucks the csv up or else) + # if OK return True + #if not : return False + if desc == "": + # empty description is fine as it's optional + return True + pattern = re.compile("^[A-Za-z0-9-.,' \"]+$") + desc = str(desc) + desc.strip() + if pattern.fullmatch(desc) is None: + return False + if desc == "DEFAULT": + return False + elif len(desc) > 256: + #print("desc is greater than 256 chars") + return False + return True + +def IsCategoryValid(categories: list)-> bool: + """ + Check the categories are only [a-zA-Z0-9 ] with 64 max chars. + """ + # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max + #(careful with the ' and , make sure you test if it fucks the csv up or else) + # if OK return True + #if not : return False + pattern = re.compile("^[A-Za-z0-9 ]+$") + for category in categories: + category.strip() + if pattern.fullmatch(category) is None: + #print('Got an empty list or invalid chars') + return False + elif len(category) > 64: + #print('Category is too long') + return False + else: + return True + +def IsNameValid(name: str)->bool: + """ + Check the parameter name only contains [a-zA-Z0-9 ] and is 64 chars long. + """ + # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max + #(careful with the ' and , make sure you test if it fucks the csv up or else) + # if OK return True + #if not : return False + pattern = re.compile("^[A-Za-z0-9 ]+$") + name = name.strip() + if (pattern.fullmatch(name) is None): + #print("Got an invalid character or nothing") + return False + elif len(name) > 64: + #print(f'Got a name length greater than 64. {len(name)}') + return False + return True + + + +def print_colors(s:str, bold=False, is_error = False, default=False): + """ + Helper function to print with colors + """ + if is_error: + print(f"{RED}{s}{RESET}") + elif bold: + print(f"{BOLD_PURPLE}{s}{RESET}") + elif is_error and bold: + print(f"{BOLD_RED}{s}{RESET}") + elif default: + print(f'{s}') + else: + print(f"{PURPLE}{s}{RESET}") + \ No newline at end of file