mirror of
http://git.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/nihilist/darknet-lantern.git
synced 2025-05-16 04:06:59 +00:00
1038 lines
64 KiB
Python
1038 lines
64 KiB
Python
from utils import *
|
|
|
|
import os, pwd
|
|
import pandas as pd
|
|
import requests
|
|
import shutil
|
|
import time
|
|
import urllib
|
|
import sys
|
|
|
|
|
|
def main():
|
|
#os.system('clear')
|
|
proxies = {
|
|
'http': 'socks5h://127.0.0.1:9050',
|
|
'https': 'socks5h://127.0.0.1:9050'
|
|
}
|
|
|
|
rootpath='/srv/darknet-lantern/'
|
|
urlpath=pwd.getpwuid(os.getuid()).pw_dir+"/.darknet_participant_url"
|
|
participantsdir=rootpath+'www/participants/'
|
|
officialparticipants=rootpath+'www/.official_participants'
|
|
|
|
# check if /srv/darknet-lantern/www/participants directory exists,
|
|
if not os.path.isdir(participantsdir):
|
|
print("participants directory doesnt exist, creating it")
|
|
os.makedirs(participantsdir)
|
|
# iterate over /srv/darknet-lantern/www/.official_participants, list each line
|
|
with open(officialparticipants, 'r') as file:
|
|
# for each line (which is a participant):
|
|
for line in file:
|
|
participantdir=participantsdir+line.strip()
|
|
# check if the directory exists
|
|
if not os.path.isdir(participantdir):
|
|
#if not, create it
|
|
print("Official participan ",line.strip() , "'s directory doesnt exist, creating it")
|
|
os.makedirs(participantdir)
|
|
|
|
|
|
|
|
|
|
|
|
print_colors("""
|
|
;
|
|
ED.
|
|
E#Wi G: L. ,;
|
|
E###G. j. E#, :EW: ,ft f#i
|
|
E#fD#W; .. EW, E#t .GEE##; t#E .E#t GEEEEEEEL
|
|
E#t t##L ;W, E##j E#t j#K;E###t t#E i#W, ,;;L#K;;.
|
|
E#t .E#K, j##, E###D. E#GK#f E#fE#f t#E L#D. t#E
|
|
E#t j##f G###, E#jG#W; E##D. E#t D#G t#E :K#Wfff; t#E
|
|
E#t :E#K: :E####, E#t t##f E##Wi E#t f#E. t#E i##WLLLLt t#E
|
|
E#t t##L ;W#DG##, E#t :K#E: E#jL#D: E#t t#K: t#E .E#L t#E
|
|
E#t .D#W; j###DW##, E#KDDDD###iE#t ,K#jE#t ;#W,t#E f#E: t#E
|
|
E#tiW#G. G##i,,G##, E#f,t#Wi,,,E#t jDE#t :K#D#E ,WW; t#E
|
|
E#K##i :K#K: L##, E#t ;#W: j#t E#t .E##E .D#; t#E
|
|
E##D. ;##D. L##, DWi ,KK: ,; .. G#E tt fE
|
|
E#t ,,, .,, fE :
|
|
L: ,
|
|
|
|
|
|
L. ,; L.
|
|
i EW: ,ft f#i j. EW: ,ft
|
|
LE .. E##; t#E GEEEEEEEL .E#t EW, E##; t#E
|
|
L#E ;W, E###t t#E ,;;L#K;;. i#W, E##j E###t t#E
|
|
G#W. j##, E#fE#f t#E t#E L#D. E###D. E#fE#f t#E
|
|
D#K. G###, E#t D#G t#E t#E :K#Wfff; E#jG#W; E#t D#G t#E
|
|
E#K. :E####, E#t f#E. t#E t#E i##WLLLLt E#t t##f E#t f#E. t#E
|
|
.E#E. ;W#DG##, E#t t#K: t#E t#E .E#L E#t :K#E: E#t t#K: t#E
|
|
.K#E j###DW##, E#t ;#W,t#E t#E f#E: E#KDDDD###iE#t ;#W,t#E
|
|
.K#D G##i,,G##, E#t :K#D#E t#E ,WW; E#f,t#Wi,,,E#t :K#D#E
|
|
.W#G :K#K: L##, E#t .E##E t#E .D#; E#t ;#W: E#t .E##E
|
|
:W##########Wt ;##D. L##, .. G#E fE tt DWi ,KK: .. G#E
|
|
:,,,,,,,,,,,,,.,,, .,, fE : fE
|
|
, ,
|
|
version: 1.0.1
|
|
""", bold=True)
|
|
|
|
|
|
while True:
|
|
if os.path.isfile(urlpath):
|
|
with open(urlpath) as f:
|
|
instance = f.read().rstrip()
|
|
if IsOnionValid(instance):
|
|
print_colors(f"[+] Instance Name: {instance}. Valid:{IsOnionValid(instance)}")
|
|
break
|
|
else:
|
|
print_colors(f'[-] Invalid instance name in ~/.darknet_participant_url: {instance}',is_error=True )
|
|
break
|
|
else:
|
|
print_colors("[+] Instance Path doesn't exist yet")
|
|
print_colors(f"Your url will be saved here {urlpath}")
|
|
instance = input("What is your Instance domain?(ex: lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion): ")
|
|
if IsOnionValid(instance):
|
|
print_colors(f"[+] Instance Name: {instance}. Valid: {IsUrlValid(instance)}")
|
|
instancepath=rootpath+'www/participants/'+instance
|
|
else:
|
|
print_colors(f'[-] Invalid instance name in ~/.darknet_participant_url: {instance}', is_error=True )
|
|
break
|
|
|
|
isitvalid=input("Is this your this your instance domain?(y/n) ")
|
|
if isitvalid == "y" :
|
|
print_colors("OK writing the instance url to ~/.darknet_participants_url")
|
|
with open(urlpath, "w") as file:
|
|
file.write(instance)
|
|
print_colors("[+] File written")
|
|
f = open(urlpath,"r")
|
|
print_colors(f"{f.read()}")
|
|
print_colors("[+] Initial Setup Completed!")
|
|
myinstance = instance
|
|
|
|
instancepath=rootpath+'www/participants/'+instance
|
|
templatepath=rootpath+'templates/'
|
|
verifiedcsvfile=instancepath+'/verified.csv'
|
|
unverifiedcsvfile=instancepath+'/unverified.csv'
|
|
blcsvfile=instancepath+'/blacklist.csv'
|
|
secsvfile=instancepath+'/sensitive.csv'
|
|
webpcsvfile=instancepath+'/webring-participants.csv'
|
|
if not os.path.exists(instancepath):
|
|
print_colors(f"{rootpath}",is_error=True, bold=True)
|
|
os.makedirs(instancepath)
|
|
# check if all the required csv files exist in it, otherwise copy them from the templates directory
|
|
# NOTE : the templates files are EMPTY by default, this is because each peer has to manually review lists of links, and add links themselves manually, this is to avoid allowing malicious links to slip through without intentional edits from the peer themselves.
|
|
for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv','banner.png']:
|
|
filepath=instancepath+'/'+i
|
|
if not os.path.isfile(filepath):
|
|
# copy templates/ FILE.CSV to instancepath/ FILE.CSV
|
|
src=templatepath+i
|
|
shutil.copyfile(src, filepath)
|
|
# now that they exist, get vdf and uvdf and the rest
|
|
vdf = pd.read_csv(verifiedcsvfile)
|
|
uvdf = pd.read_csv(unverifiedcsvfile)
|
|
bldf = pd.read_csv(blcsvfile)
|
|
sedf = pd.read_csv(secsvfile)
|
|
webpdf = pd.read_csv(webpcsvfile)
|
|
print_colors(f"[+] file exists, your Webring URL is {instance}")
|
|
|
|
##### CHECK IF ARGUMENTS ARE PASSED TO ENTER PROMPT-LESS MODE #####
|
|
if len(sys.argv) == 2 and sys.argv[1] == "4":
|
|
print("4) Synchronize new links from existing webring participants into your unverified.csv file")
|
|
option=4
|
|
elif len(sys.argv) == 2 and sys.argv[1] == "9":
|
|
print("remove duplicate urls from instance")
|
|
option=9
|
|
elif len(sys.argv) == 2 and sys.argv[1] == "10":
|
|
print("Perform sanity checks on all csv files for all instances")
|
|
option=10
|
|
else:
|
|
print_colors("""
|
|
[+] Welcome to your own Darknet Lantern Instance, where you can explore the Darknet and help others do the same.
|
|
|
|
Managing Websites:
|
|
1) Add a new Website entry (into unverified.csv)
|
|
2) Trust/Untrust/ Blacklist(WIP) a Website entry (move an entry from unverified to verified.csv)
|
|
3) Edit link attributes (WIP)
|
|
|
|
Managing Webring Participants:
|
|
4) Synchronize new links from existing webring participants, into your unverified.csv file
|
|
5) Add a new webring participant (and download their files into their directory (without trusting them yet!))
|
|
6) Trust/UnTrust/Blacklist a webring participant (Potentially dangerous)
|
|
|
|
Managing Wordlists:
|
|
7) Add/Remove Words/URLs in the sensitive list (ex: drug)
|
|
8) Add/Remove Words/URLs or links in the blacklist (ex: porn)
|
|
|
|
Maintenance:
|
|
9) Remove the duplicate URLs for your own instance
|
|
10) Perform sanity checks on all csv files for all instances (to mark them as sensitive / or remove the ones that are blacklisted)
|
|
|
|
0) Exit
|
|
""")
|
|
option = input("Select an option? (0-10): ").strip()
|
|
try:
|
|
option = int(option)
|
|
except ValueError:
|
|
print_colors(f"[-] Exiting. {option} is not a valid option.", bold=True, is_error=True)
|
|
return False
|
|
while True:
|
|
match option:
|
|
|
|
########## MANAGING WEBSITE ENTRIES #################
|
|
#Websites:
|
|
# 1) Add a new Website entry (into unverified.csv)
|
|
# 2) Trust a Website entry (move an entry from unverified to verified.csv)
|
|
# 3) Untrust a Website entry (move an entry from unverified to verified.csv)
|
|
#####################################################
|
|
case 1:
|
|
while True:
|
|
print_colors("\n[+] Add a new Website entry (into unverified.csv)")
|
|
name=''
|
|
while(IsNameValid(name) is not True):
|
|
name = input("What is the name of the website? ")
|
|
category=''
|
|
while(IsCategoryValid(category) is not True):
|
|
category = input("What is the website Category? ")
|
|
# the url of the website (required) + check if its valid
|
|
url=''
|
|
while(IsUrlValid(url) is not True):
|
|
url=input("What is the website URL ? ")
|
|
|
|
# a quick description (optional) + check if its valid
|
|
desc='DEFAULT'
|
|
while(IsDescriptionValid(desc) is not True):
|
|
desc=input("Description for the website ? (if not empty = the link will be added to verified.csv directly ) ")
|
|
choice=input("Is the website sensitive ? (ex: related to drugs) (y/n) ")
|
|
if choice == "n":
|
|
sensi = 'NO'
|
|
else:
|
|
sensi = 'YES'
|
|
|
|
newrow=[instance,category,name,url,sensi,desc,'YES','100']
|
|
print_colors(f"[+] NEWROW= {newrow}")
|
|
# (rest is automatic: status, score, instance is = '' because it is your own instance)
|
|
# TODO check if the entry doesnt already exist in verified.csv and in unverified.csv
|
|
# if it doesnt exist, add it into unverified.csv
|
|
if desc == '': # if the description is empty = it means that it goes in unverified.csv
|
|
print("Adding new row in unverified.csv since description is empty")
|
|
uvdf.loc[-1] = newrow # adding a row
|
|
uvdf.index = uvdf.index + 1 # shifting index
|
|
uvdf = uvdf.sort_index() # sorting by index
|
|
uvdf = uvdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
|
|
print_colors("[+] New row added! now writing the csv file")
|
|
uvdf.to_csv(unverifiedcsvfile, index=False)
|
|
else: # if the description isnt empty = it means that it goes in verified.csv
|
|
print("Adding new row in verified.csv since description is not empty")
|
|
vdf.loc[-1] = newrow # adding a row
|
|
vdf.index = vdf.index + 1 # shifting index
|
|
vdf = vdf.sort_index() # sorting by index
|
|
vdf = vdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
|
|
print_colors("[+] New row added! now writing the csv file")
|
|
vdf.to_csv(verifiedcsvfile, index=False)
|
|
|
|
choice=input("\n[+] Want to add another website ? (y/n) ")
|
|
if choice == "n":
|
|
break
|
|
break
|
|
|
|
|
|
case 2:
|
|
print_colors("[+] Trust/Untrust/Blacklist a Website entry (move an entry from unverified to verified.csv)")
|
|
while True:
|
|
vdf = pd.read_csv(verifiedcsvfile)
|
|
uvdf = pd.read_csv(unverifiedcsvfile)
|
|
# search for a word
|
|
name=''
|
|
|
|
#ask the user if they want to 1) trust, 2) untrust 3) blacklist the selected website
|
|
choice = int(input("Do you want to 1) Trust, 2) UnTrust, or 3) Blacklist an existing entry ?").strip())
|
|
while True:
|
|
match choice:
|
|
case 1:
|
|
print_colors(f"{uvdf[['Name','URL']]}")
|
|
while(IsNameValid(name) is not True):
|
|
name = input("What is the Website name you want to Trust ? (ex: Nowhere)")
|
|
filter_uvdf = uvdf[uvdf.Name.str.contains(name)]
|
|
# display only the matching entries in unverified.csv in an array format (display it in CLI).
|
|
print_colors(f"{filter_uvdf[['Name','URL']]}")
|
|
# check if there are no results, dont proceed if there are none!
|
|
if filter_uvdf.size == 0:
|
|
print_colors("ERROR no results, skipping.",is_error=True)
|
|
else:
|
|
# Each of the rows has an index,
|
|
index=-1
|
|
while (index not in filter_uvdf.index):
|
|
index = int(input("What is the index of the entry that you want to move to Trust ? (ex: 3) "))
|
|
# once selected, it must be able to SAVE and print_colors that row:
|
|
print_colors(f"{uvdf.iloc[index].values}")
|
|
|
|
# 1) Trust an existing website (move it from unverified.csv to verified.csv)
|
|
newrow=uvdf.iloc[index].values
|
|
newdesc=""
|
|
#while the description is empty, keep looping
|
|
while(newdesc == "" or (IsDescriptionValid(newdesc) is not True)):
|
|
newdesc=input("Description for the website ? (it cannot be empty, for the link to be added in verified.csv): ")
|
|
|
|
# append it into verified.csv
|
|
vdf.loc[-1] = newrow # adding a row
|
|
vdf.at[-1, 'Description']=newdesc
|
|
vdf.index = vdf.index + 1 # shifting index
|
|
vdf = vdf.sort_index() # sxorting by index
|
|
vdf = vdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
|
|
vdf.to_csv(verifiedcsvfile, index=False)
|
|
print_colors("[+] New row added to verified.csv! now writing to the csv")
|
|
|
|
# remove it from unverified.csv
|
|
uvdf.drop(index, inplace= True)
|
|
uvdf = uvdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
|
|
uvdf.to_csv(unverifiedcsvfile, index=False)
|
|
print_colors("[+] Link is now moved to verified.csv!")
|
|
break
|
|
case 2:
|
|
print_colors(f"{vdf[['Name','URL']]}")
|
|
while(IsNameValid(name) is not True):
|
|
name = input("What is the Website name you want to Untrust ? (ex: BreachForums)")
|
|
filter_vdf = vdf[vdf.Name.str.contains(name)]
|
|
# display only the matching entries in unverified.csv in an array format (display it in CLI).
|
|
print_colors(f"{filter_vdf[['Name','URL']]}")
|
|
# check if there are no results, dont proceed if there are none!
|
|
if filter_vdf.size == 0:
|
|
print_colors("ERROR no results, skipping.",is_error=True)
|
|
else:
|
|
# Each of the rows has an index,
|
|
index=-1
|
|
while (index not in filter_vdf.index):
|
|
index = int(input("What is the index of the entry that you want to move to Untrust ? (ex: 3) "))
|
|
# once selected, it must be able to SAVE and print_colors that row:
|
|
print_colors(f"{uvdf.iloc[index].values}")
|
|
|
|
# 1) Untrust an existing website (move it from verified.csv to unverified.csv)
|
|
newrow=vdf.iloc[index].values
|
|
|
|
# append it into unverified.csv
|
|
uvdf.loc[-1] = newrow # adding a row
|
|
uvdf.index = uvdf.index + 1 # shifting index
|
|
uvdf = uvdf.sort_index() # sxorting by index
|
|
uvdf = uvdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
|
|
uvdf.to_csv(unverifiedcsvfile, index=False)
|
|
print_colors("[+] New row added to unverified.csv! now writing to the csv")
|
|
|
|
# remove it from verified.csv
|
|
vdf.drop(index, inplace= True)
|
|
vdf = vdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
|
|
vdf.to_csv(verifiedcsvfile, index=False)
|
|
print_colors("[+] Link is now moved to unverified.csv!")
|
|
break
|
|
case 3:
|
|
#TODO: 3) Blacklist an existing website
|
|
print_colors("[+] Blacklisting link!")
|
|
while(IsNameValid(name) is not True):
|
|
name = input("What is the Website name you want to Blacklist ? (ex: BreachForums)")
|
|
filter_uvdf = uvdf[uvdf.Name.str.contains(name)]
|
|
filter_vdf = vdf[vdf.Name.str.contains(name)]
|
|
#TODO: put the URL of the website in your blacklist.csv file
|
|
#TODO: drop the rows that contain that URL in your verified.csv and unverified.csv
|
|
break
|
|
choice=input("\n[+] Want to Trust/Untrust/Blacklist another existing entry ? (y/n) ")
|
|
if choice == "n":
|
|
break
|
|
break
|
|
|
|
case 3:
|
|
# TODO change it to "edit link attributes"
|
|
print_colors("[+] Edit link attributes (WIP)")
|
|
break
|
|
|
|
|
|
####### MANAGING WEBRING PARTICIPANTS ###########
|
|
# 4) Synchronize new links from webring participants, into your unverified.csv file
|
|
# 5) Add a new webring participant (and download their files into their directory (without trusting them yet!))
|
|
# 6) Trust/UnTrust/Blacklist a webring participant
|
|
#####################################################
|
|
|
|
|
|
|
|
#check if it works when you have a second webring participant
|
|
case 4:
|
|
print_colors("4) Synchronize new links from existing webring participants, into your unverified.csv file")
|
|
participantsdir=rootpath+'www/participants/'
|
|
name=''
|
|
desc=''
|
|
trusted=''
|
|
status=''
|
|
score=''
|
|
webringcsvfile=instancepath+'/'+'webring-participants.csv'
|
|
wdf = pd.read_csv(webringcsvfile)
|
|
for participant in os.listdir(participantsdir):
|
|
participantdir=participantsdir+participant
|
|
|
|
# NOTE check if the webring participant is yourself, if it is, then skip it
|
|
if participant != myinstance: # prod: dont use your own intance
|
|
#if participant == myinstance: # preprod testing only on your own instance
|
|
#overwrite the existing files in the participant's directory, with their version (download all the csv files from them again)
|
|
basewurl='http://'+participant+'/participants/'+participant+'/'
|
|
print_colors(f"{basewurl}")
|
|
print_colors(f"[+] Downloading the files of: {participant} ")
|
|
w_vcsv=basewurl+'verified.csv'
|
|
w_uvcsv=basewurl+'unverified.csv'
|
|
w_blcsv=basewurl+'blacklist.csv'
|
|
w_scsv=basewurl+'sensitive.csv'
|
|
w_webcsv=basewurl+'webring-participants.csv'
|
|
|
|
# verify that their verified.csv csv file exists at basewurl+'verified.csv'
|
|
if CheckUrl(w_vcsv) is False or CheckUrl(w_uvcsv) is False or CheckUrl(w_blcsv) is False or CheckUrl(w_scsv) is False or CheckUrl(w_webcsv) is False:
|
|
print_colors("[-] Webring Participant isn't reachable, skipping", is_error=True)
|
|
else: #if the webring participant is reachable, proceed
|
|
print_colors("[+] Webring Participant is reachable, updating their csv files:")
|
|
for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv']:
|
|
# FOR EACH CSV FILE TO GET:
|
|
# URL: basewurl / FILE.CSV
|
|
# PATH: participantdir / FILE.CSV
|
|
# download the external csv file and save it into the "text" variable:
|
|
#response = urllib.request.urlopen(basewurl+i)
|
|
response = requests.get(basewurl+i, proxies=proxies)
|
|
#data = response.read() # a `bytes` object
|
|
#text = data.decode('utf-8')
|
|
text = response.text
|
|
# save the text variable into the destination file:
|
|
csvfilepath=participantdir+'/'+i
|
|
with open(csvfilepath, "w") as file:
|
|
file.write(text)
|
|
f = open(csvfilepath,"r")
|
|
|
|
# download the banner.png image:
|
|
|
|
bannerurl=basewurl+'banner.png'
|
|
bannerpath=participantdir+'/banner.png'
|
|
r = requests.get(bannerurl, stream=True, proxies=proxies)
|
|
with open(bannerpath, 'wb') as f:
|
|
r.raw.decode_content = True
|
|
shutil.copyfileobj(r.raw, f)
|
|
|
|
# SANITY CHECK ON THE BANNER PNG IMAGE:
|
|
if IsBannerValid(bannerpath):
|
|
pass
|
|
else:
|
|
# if false, overwrite it with the template banner png file
|
|
os.remove(bannerpath)
|
|
# copy templates/banner.png to bannerpath
|
|
bannertemplatepath=templatepath+'banner.png'
|
|
shutil.copyfile(bannertemplatepath, bannerpath)
|
|
|
|
|
|
# check if the participant is already listed in webring-participants.csv or not, and add them if not already listed
|
|
# and display only the matching entries in unverified.csv in an array format (display it in CLI).
|
|
filter_wdf = wdf[wdf.URL.str.contains(participant)]
|
|
# check if there are no results, dont proceed if there are none!
|
|
if filter_wdf.size == 0: #skip if webring participant is already listed, otherwise proceed
|
|
newrow=[name,participant,desc,trusted,status,score]
|
|
wdf.loc[-1] = newrow # adding a row
|
|
wdf.index = wdf.index + 1 # shifting index
|
|
wdf = wdf.sort_index() # sorting by index
|
|
wdf.to_csv(webringcsvfile, index=False)
|
|
else:
|
|
pass
|
|
|
|
# iterate through the participant's verified.csv and unverified.csv files
|
|
for w in ['verified.csv','unverified.csv']:
|
|
csvfilepath=participantdir+'/'+w
|
|
print_colors(f"{csvfilepath}")
|
|
csvdf = pd.read_csv(csvfilepath)
|
|
bldf[['blacklisted-words']].iterrows()
|
|
rows2delete= [] # it is an empty list at first
|
|
for i,j in csvdf.iterrows():
|
|
row=csvdf.loc[i,:].values.tolist()
|
|
#print_colors(f"{row}")
|
|
|
|
|
|
|
|
################################ SANITY CHECKS ####################################
|
|
### SANITY CHECK 0: make sure that ✔️ and x are replaced with YES/NO, as it changed since v1.0.1 ###
|
|
if csvdf.at[i, 'Status'] == "✔️" or csvdf.at[i, 'Status'] == "YES" :
|
|
csvdf.at[i, 'Status'] = "YES"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
else:
|
|
csvdf.at[i, 'Status'] = "NO"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
|
|
if csvdf.at[i, 'Sensitive'] == "✔️" or csvdf.at[i, 'Sensitive'] == "YES" :
|
|
csvdf.at[i, 'Sensitive'] = "YES"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
else:
|
|
csvdf.at[i, 'Sensitive'] = "NO"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
|
|
### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion###
|
|
if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False:
|
|
#mark the row for deletion as it has invalid inputs
|
|
if i not in rows2delete:
|
|
print_colors(f"Marking row {i} for deletion, as it has invalid inputs")
|
|
rows2delete.append(i) #mark the row for deletion if not already done
|
|
|
|
### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ###
|
|
for k,l in bldf.iterrows():
|
|
blword=bldf.at[k, 'blacklisted-words']
|
|
if any(blword in str(x) for x in row) == True:
|
|
if i not in rows2delete:
|
|
print_colors(f"Marking row {i} for deletion, as it matches with a blacklisted word")
|
|
rows2delete.append(i) #mark the row for deletion if not already done
|
|
else:
|
|
if i not in rows2delete:
|
|
# not a blacklisted link, therefore it is suitable to be added to your own csv files:
|
|
################################ CHECKING FOR DUPLICATES! #########################
|
|
# for each link in the participant's verified/unverified csv files,
|
|
# check if the link is already listed in your own verified.csv or unverified.csv
|
|
filterterm=csvdf.at[i, 'URL']
|
|
filter_vdf= vdf[vdf.URL.str.contains(filterterm)]
|
|
filter_uvdf= uvdf[uvdf.URL.str.contains(filterterm)]
|
|
if len(filter_uvdf.index) == 0 and len(filter_vdf.index) == 0:
|
|
newrow=row
|
|
uvdf.loc[-1] = newrow # adding a row
|
|
uvdf.index = uvdf.index + 1 # shifting index
|
|
uvdf = uvdf.sort_index() # sorting by index
|
|
uvdf.to_csv(unverifiedcsvfile, index=False)
|
|
print_colors("[+] New row added to your own unverified.csv file!")
|
|
else:
|
|
pass
|
|
#print_colors(f'[-] Skipping row as it is already added in {w} {row}',is_error=True)
|
|
|
|
|
|
|
|
###################### APPENDING TO YOUR OWN UNVERIFIED.CSV FILE###################
|
|
|
|
|
|
### SANITY CHECK 3: Mark all the rows that are supposed to be sensitive ###
|
|
for k,l in sedf.iterrows():
|
|
seword=sedf.at[k, 'sensitive-words']
|
|
if any(seword in str(x) for x in row) == True:
|
|
if csvdf.at[i, 'Sensitive'] != 'NO':
|
|
print_colors(f"Marking row {i} as sensitive, as it matches with a sensitive word")
|
|
csvdf.at[i, 'Sensitive']='YES'
|
|
|
|
#print_colors(f'[-] Rows to delete: {rows2delete}', is_error=True)
|
|
# only delete rows after you've gone through all the unverified.csv OR verified.csv rows'
|
|
for i in rows2delete:
|
|
row=csvdf.loc[i,:].values.tolist()
|
|
print_colors(f'[+] REMOVING ROW: {i}{row}')
|
|
csvdf.drop(i, inplace= True)
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
rows2delete= [] # it is an empty list at first
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
case 5:
|
|
print_colors("[+] Add a new webring participant (and download their files into their directory (without trusting them yet!))")
|
|
webring_participant_url = ''
|
|
while(IsOnionValid(webring_participant_url) is not True):
|
|
webring_participant_url = input("What is the onion domain of the new webring participant? (ex: lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion) ")
|
|
participantdir=rootpath+'www/participants/'+webring_participant_url
|
|
if os.path.isdir(participantdir):
|
|
print_colors("[-] Webring Participant is already listed, skipping.")
|
|
else:
|
|
basewurl='http://'+webring_participant_url+'/participants/'+webring_participant_url+'/'
|
|
print_colors(f"{basewurl}")
|
|
print_colors(f"[+] Checking if all of the required csv files exists for new webring participant {webring_participant_url} : ")
|
|
w_vcsv=basewurl+'verified.csv'
|
|
w_uvcsv=basewurl+'unverified.csv'
|
|
w_blcsv=basewurl+'blacklist.csv'
|
|
w_scsv=basewurl+'sensitive.csv'
|
|
w_webcsv=basewurl+'webring-participants.csv'
|
|
|
|
# verify that their verified.csv csv file exists at basewurl+'verified.csv'
|
|
if CheckUrl(w_vcsv) is False or CheckUrl(w_uvcsv) is False or CheckUrl(w_blcsv) is False or CheckUrl(w_scsv) is False or CheckUrl(w_webcsv) is False:
|
|
print_colors("[-] Webring Participant is invalid, exiting.")
|
|
else:
|
|
print_colors("[+] Webring Participant is valid, adding it.")
|
|
name=''
|
|
while(IsNameValid(name) is not True):
|
|
name = input("What is the Webring instance name ? ")
|
|
desc='DEFAULT'
|
|
while(IsDescriptionValid(desc) is not True):
|
|
desc=input("Description for the webring participant ? (Optional)")
|
|
trusted=''
|
|
status=''
|
|
score=''
|
|
newrow=[name,webring_participant_url,desc,trusted,status,score]
|
|
webringcsvfile=instancepath+'/'+'webring-participants.csv'
|
|
wdf = pd.read_csv(webringcsvfile)
|
|
wdf.loc[-1] = newrow # adding a row
|
|
wdf.index = wdf.index + 1 # shifting index
|
|
wdf = wdf.sort_index() # sorting by index
|
|
print_colors(f"[+] New row added! now writing the csv file: {webringcsvfile}")
|
|
wdf.to_csv(webringcsvfile, index=False)
|
|
|
|
if not os.path.exists(participantdir):
|
|
os.makedirs(participantdir)
|
|
for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv']:
|
|
# FOR EACH CSV FILE TO GET:
|
|
# URL: basewurl / FILE.CSV
|
|
# PATH: participantdir / FILE.CSV
|
|
print_colors(f'[+] DOWNLOADING {basewurl}{i}')
|
|
response = requests.get(basewurl+i, proxies=proxies)
|
|
text = response.text
|
|
print_colors(f"[+] SAVING IT INTO participantdir/{i}")
|
|
csvfilepath=participantdir+'/'+i
|
|
with open(csvfilepath, "w") as file:
|
|
file.write(text)
|
|
print_colors("[+] file written, let's read it")
|
|
f = open(csvfilepath,"r")
|
|
print_colors(f.read())
|
|
|
|
# download the banner.png image:
|
|
|
|
bannerurl=basewurl+'banner.png'
|
|
bannerpath=participantdir+'/banner.png'
|
|
r = requests.get(bannerurl, stream=True, proxies=proxies)
|
|
with open(bannerpath, 'wb') as f:
|
|
r.raw.decode_content = True
|
|
shutil.copyfileobj(r.raw, f)
|
|
|
|
# SANITY CHECK ON THE BANNER PNG IMAGE:
|
|
if IsBannerValid(bannerpath):
|
|
print_colors('[+] Banner is valid')
|
|
else:
|
|
# if false, overwrite it with the template banner png file
|
|
print_colors('[-] Banner is not valid, replacing it with the default banner')
|
|
os.remove(bannerpath)
|
|
# copy templates/banner.png to bannerpath
|
|
bannertemplatepath=templatepath+'banner.png'
|
|
shutil.copyfile(bannertemplatepath, bannerpath)
|
|
|
|
########### PERFORM SANITY CHECKS ON the webring participant's verified.csv and unverified.csv ##################
|
|
for w in ['verified.csv','unverified.csv']:
|
|
csvfilepath=participantdir+'/'+w
|
|
csvdf = pd.read_csv(csvfilepath)
|
|
|
|
#print_colors(bldf[['blacklisted-words']])
|
|
bldf[['blacklisted-words']].iterrows()
|
|
rows2delete= [] # it is an empty list at first
|
|
for i,j in csvdf.iterrows():
|
|
#row=uvdf.iloc[[i]] #it displays the index
|
|
row=csvdf.loc[i,:].values.tolist()
|
|
|
|
################################ SANITY CHECKS ####################################
|
|
### SANITY CHECK 0: make sure that ✔️ and x are replaced with YES/NO, as it changed since v1.0.1 ###
|
|
if csvdf.at[i, 'Status'] == "✔️" or csvdf.at[i, 'Status'] == "YES" :
|
|
csvdf.at[i, 'Status'] = "YES"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
else:
|
|
csvdf.at[i, 'Status'] = "NO"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
|
|
if csvdf.at[i, 'Sensitive'] == "✔️" or csvdf.at[i, 'Sensitive'] == "YES" :
|
|
csvdf.at[i, 'Sensitive'] = "YES"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
else:
|
|
csvdf.at[i, 'Sensitive'] = "NO"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
|
|
### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion###
|
|
if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False:
|
|
#mark the row for deletion as it has invalid inputs
|
|
if i not in rows2delete:
|
|
print_colors(f"Marking row {i} for deletion, as it has invalid inputs")
|
|
rows2delete.append(i) #mark the row for deletion if not already done
|
|
|
|
### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ###
|
|
for k,l in bldf.iterrows():
|
|
blword=bldf.at[k, 'blacklisted-words']
|
|
if any(blword in str(x) for x in row) == True:
|
|
if i not in rows2delete:
|
|
print_colors(f"Marking row {i} for deletion, as it matches with a blacklisted word")
|
|
rows2delete.append(i) #mark the row for deletion if not already done
|
|
|
|
### SANITY CHECK 3: Mark all the rows that are supposed to be sensitive ###
|
|
for k,l in sedf.iterrows():
|
|
seword=sedf.at[k, 'sensitive-words']
|
|
if any(seword in str(x) for x in row) == True:
|
|
if csvdf.at[i, 'Sensitive'] != 'NO':
|
|
print_colors(f"Marking row {i} as sensitive, as it matches with a sensitive word")
|
|
csvdf.at[i, 'Sensitive']='YES'
|
|
|
|
#print_colors(f"[-] Rows to delete: {rows2delete}")
|
|
|
|
for i in rows2delete:
|
|
row=csvdf.loc[i,:].values.tolist()
|
|
print_colors(f"[+] REMOVING ROW: {i}{row}")
|
|
csvdf.drop(i, inplace= True)
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
break
|
|
##############################################
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
case 6:
|
|
while True:
|
|
print_colors("[+] Trust/UnTrust/Blacklist a webring participant (Potentially dangerous)")
|
|
webringcsvfile=instancepath+'/'+'webring-participants.csv'
|
|
wdf = pd.read_csv(webringcsvfile)
|
|
print_colors(f'{wdf[["URL","Trusted"]]}')
|
|
try:
|
|
index = int(input("What is the index of the webring participant that you want to edit? -1 to exit ").strip())
|
|
if index == -1:
|
|
break
|
|
elif index in wdf.index:
|
|
choice = int(input("Do you want to 1) Trust, 2) UnTrust, or 3) Blacklist the webring participant?").strip())
|
|
while True:
|
|
match choice:
|
|
case 1:
|
|
# trust the webring participant
|
|
choice2=input("You're about to trust another peer, this means that you're going to automatically trust all of the links they have in their verified.csv file! If this is a malicious peer, this action might be potentially risky! Do you want to continue ? (y/n)")
|
|
if choice2 == "y":
|
|
print_colors(f'[+] Trusting webring participant {wdf.at[index,"URL"]}')
|
|
## Warning: In future versions of panda '✔️' will not work. It will show an error.
|
|
wdf.at[index,"Trusted"]= 'YES'
|
|
wdf.to_csv(webringcsvfile, index=False)
|
|
break
|
|
else:
|
|
print_colors("[-] not trusting webring participant", is_error=True)
|
|
break
|
|
|
|
case 2:
|
|
print_colors(f'[+] UnTrusting webring participant {wdf.at[index,"URL"]}')
|
|
## Warning: In future versions of panda '' will not work. It will show an error. Maybe change to a 0,1
|
|
wdf.at[index,"Trusted"]='NO'
|
|
wdf.to_csv(webringcsvfile, index=False)
|
|
break
|
|
|
|
case 3:
|
|
print_colors(f'[+] Blacklisting webring participant {wdf.at[index,"URL"]}')
|
|
instance2blacklist=wdf.at[index,"URL"]
|
|
newrow=[instance2blacklist]
|
|
print_colors(f"[+] NEWROW= {newrow}")
|
|
# (rest is automatic: status, score, instance is = '' because it is your own instance)
|
|
# check if the entry doesn't already exist in verified.csv and in unverified.csv
|
|
# if it doesnt exist, add it into unverified.csv
|
|
bldf.loc[-1] = newrow # adding a row
|
|
bldf.index = bldf.index + 1 # shifting index
|
|
bldf = bldf.sort_index() # sorting by index
|
|
print_colors("[+] New row added! now writing the csv file:")
|
|
bldf.to_csv(blcsvfile, index=False)
|
|
|
|
|
|
# remove all of the entries that came from that participant (drop the lines in your own verified+unverified.csv that have that instance in the instance column)
|
|
|
|
rows2delete= [] # it is an empty list at first
|
|
for i,j in vdf.iterrows():
|
|
row=vdf.loc[i,:].values.tolist()
|
|
for k,l in bldf.iterrows():
|
|
blword=bldf.at[k, 'blacklisted-words']
|
|
if any(blword in str(x) for x in row) == True:
|
|
if i not in rows2delete:
|
|
print_colors(f"Marking row {i} for deletion, as it matches with a blacklisted word")
|
|
rows2delete.append(i) #mark the row for deletion if not already done
|
|
for i in rows2delete:
|
|
row=vdf.loc[i,:].values.tolist()
|
|
print_colors(f'[+] REMOVING ROW: {i} {row}')
|
|
vdf.drop(i, inplace= True)
|
|
vdf.to_csv(verifiedcsvfile, index=False)
|
|
print_colors(f"{vdf}")
|
|
rows2delete= [] # it is an empty list at first
|
|
rows2delete= [] # it is an empty list at first
|
|
for i,j in uvdf.iterrows():
|
|
row=uvdf.loc[i,:].values.tolist()
|
|
for k,l in bldf.iterrows():
|
|
blword=bldf.at[k, 'blacklisted-words']
|
|
if any(blword in str(x) for x in row) == True:
|
|
if i not in rows2delete:
|
|
print_colors(f"Marking row {i} for deletion, as it matches with a blacklisted word")
|
|
rows2delete.append(i) #mark the row for deletion if not already done
|
|
for i in rows2delete:
|
|
row=uvdf.loc[i,:].values.tolist()
|
|
print_colors(f'[+] REMOVING ROW: {i} {row}')
|
|
uvdf.drop(i, inplace= True)
|
|
uvdf.to_csv(unverifiedcsvfile, index=False)
|
|
print_colors(f"{uvdf}")
|
|
rows2delete= [] # it is an empty list at first
|
|
|
|
# find all rows that match with the instance name in wdf aswell to remove them
|
|
for i,j in wdf.iterrows():
|
|
row=wdf.loc[i,:].values.tolist()
|
|
for k,l in bldf.iterrows():
|
|
blword=bldf.at[k, 'blacklisted-words']
|
|
if any(blword in str(x) for x in row) == True:
|
|
if i not in rows2delete:
|
|
print_colors(f"Marking row {i} for deletion, as it matches with a blacklisted word")
|
|
rows2delete.append(i) #mark the row for deletion if not already done
|
|
for i in rows2delete:
|
|
row=wdf.loc[i,:].values.tolist()
|
|
print_colors(f'[+] REMOVING ROW: {i} {row}')
|
|
wdf.drop(i, inplace= True)
|
|
wdf.to_csv(webringcsvfile, index=False)
|
|
print_colors(f"{wdf}")
|
|
rows2delete= [] # it is an empty list at first
|
|
|
|
|
|
|
|
# remove the entire directory in www/participants/INSTANCENAME aswell to get rid of it
|
|
instance2blacklistpath=rootpath+'www/participants/'+instance2blacklist
|
|
print_colors(f"[+] removing the participant's directory at {instance2blacklistpath}")
|
|
shutil.rmtree(instance2blacklistpath)
|
|
|
|
case _:
|
|
break
|
|
except Exception:
|
|
break
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
################### MANAGING WORDLISTS #################
|
|
#Managing Wordlists:
|
|
# 7) Add/Remove Words/URLs in the sensitive list (ex: drug)
|
|
# 8) Add/Remove words or links in the blacklist (ex: porn)
|
|
|
|
#Maintenance:
|
|
# 9) remove the duplicate URLs for your own instance
|
|
# 10) perform sanity checks on all csv files (all instances) (to mark them as sensitive / or remove the ones that are blacklisted)
|
|
#########################################################
|
|
|
|
case 7:
|
|
print_colors("[+] Add/Remove Words/URLs in the sensitive list (ex: drug)")
|
|
try:
|
|
option = int(input("[+] do you want to 1) add or 2) remove Words/URLs? (type -1 to exit) "))
|
|
match option:
|
|
case 1:
|
|
while True:
|
|
word=input("[+] which Sensitive word/url do you want to add? (write -1 to exit) ")
|
|
if word == "-1":
|
|
break
|
|
else:
|
|
print_colors("[+] checking if the Word/URL is valid: ")
|
|
if IsUrlValid(word) or IsOnionValid(word) or IsDescriptionValid(word):
|
|
print_colors('[+] Word/URL is valid, adding the word into the sensitive wordlist')
|
|
newrow=[word]
|
|
print_colors(f"[+] NEWROW= {newrow}")
|
|
sedf.loc[-1] = newrow
|
|
sedf.index = sedf.index + 1
|
|
sedf = sedf.sort_index()
|
|
print_colors("[+] New row added! now writing the csv file.")
|
|
sedf.to_csv(secsvfile, index=False)
|
|
|
|
|
|
case 2:
|
|
while True:
|
|
print_colors(f"{sedf}")
|
|
index=input("which word do you want to remove? (index 0 to (max index) (write -1 to exit) ")
|
|
try:
|
|
indices = index.split(' ')
|
|
if len(indices) == 2:
|
|
for i in range(int(indices[0]),int(indices[1])):
|
|
try:
|
|
idx = int(i)
|
|
if idx in sedf.index:
|
|
print_colors("[+] removing selected index.")
|
|
sedf.drop(index=idx, inplace=True)
|
|
sedf.to_csv(secsvfile, index=True)
|
|
else:
|
|
print_colors(f"[-] Index {idx} does not exist.", is_error=True)
|
|
except ValueError:
|
|
print_colors(f"[-] Error: '{i}' is not a valid integer.", is_error=True)
|
|
elif len(indices) == 1:
|
|
try:
|
|
idx = int(indices[0])
|
|
if idx != -1:
|
|
if idx in sedf.index:
|
|
print_colors("[+] removing selected index.")
|
|
sedf.drop(idx, inplace=True)
|
|
sedf.to_csv(secsvfile, index=True)
|
|
else:
|
|
print_colors(f"[-] Index {idx} does not exist.", is_error=True)
|
|
elif idx == -1:
|
|
break
|
|
except ValueError:
|
|
print_colors(f"[-] Error: '{indices[0]}' is not a valid integer.", is_error=True)
|
|
else:
|
|
print_colors('[-] Error, invalid index', is_error=True)
|
|
except Exception as e:
|
|
print_colors(f"[-] An unexpected error occurred: {str(e)}", is_error=True)
|
|
except Exception:
|
|
break
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
case 8:
|
|
print_colors("[+] Add/Remove words in the blacklist list (ex: porn)")
|
|
try:
|
|
option= int(input("[+] Do you want to 1) add or 2) remove Words/URLs? (type -1 to exit) "))
|
|
|
|
|
|
match option:
|
|
case 1:
|
|
while True:
|
|
word=input("[+] Which Sensitive word do you want to add? (write -1 to exit) ")
|
|
if word == "-1":
|
|
break
|
|
else:
|
|
print_colors("[+] Checking if the Word/URL is valid: ")
|
|
if IsUrlValid(word) or IsOnionValid(word) or IsDescriptionValid(word):
|
|
print_colors('[+] Word/URL is valid, adding the word into the blacklist')
|
|
newrow=[word]
|
|
print_colors(f"[+] NEWROW= {newrow}")
|
|
# (rest is automatic: status, score, instance is = '' because it is your own instance)
|
|
# check if the entry doesn't already exist in verified.csv and in unverified.csv
|
|
# if it doesnt exist, add it into unverified.csv
|
|
bldf.loc[-1] = newrow # adding a row
|
|
bldf.index = bldf.index + 1 # shifting index
|
|
bldf = bldf.sort_index() # sorting by index
|
|
print_colors("[+] New row added! Now writing the csv file")
|
|
bldf.to_csv(blcsvfile, index=False)
|
|
|
|
|
|
case 2:
|
|
while True:
|
|
print_colors(f"{bldf}")
|
|
index=input("which word do you want to remove? (index 0 to (max index) (write -1 to exit) ").strip()
|
|
try:
|
|
indices = index.split(' ')
|
|
if len(indices) == 2:
|
|
for i in range(int(indices[0]),int(indices[1])):
|
|
try:
|
|
idx = int(i)
|
|
if idx in bldf.index:
|
|
print_colors("[+] removing selected index.")
|
|
bldf.drop(index=idx, inplace=True)
|
|
bldf.to_csv(blcsvfile, index=True)
|
|
else:
|
|
print_colors(f"[-] Index {idx} does not exist.", is_error=True)
|
|
except ValueError:
|
|
print_colors(f"[-] Error: '{i}' is not a valid integer.", is_error=True)
|
|
elif len(indices) == 1:
|
|
try:
|
|
idx = int(indices[0])
|
|
if idx != -1:
|
|
if idx in bldf.index:
|
|
print_colors("[+] removing selected index.")
|
|
bldf.drop(idx, inplace=True)
|
|
bldf.to_csv(blcsvfile, index=False)
|
|
else:
|
|
print_colors(f"[-] Index {idx} does not exist.", is_error=True)
|
|
elif idx == -1:
|
|
break
|
|
except ValueError:
|
|
print_colors(f"[-] Error: '{indices[0]}' is not a valid integer.", is_error=True)
|
|
else:
|
|
print_colors('[-] Error, invalid index', is_error=True)
|
|
except Exception as e:
|
|
print_colors(f"[-] An unexpected error occurred: {str(e)}", is_error=True)
|
|
except Exception:
|
|
break
|
|
break
|
|
|
|
|
|
|
|
case 9:
|
|
print_colors("[+] 9) Cleaning up all duplicates in your own unverified + verified.csv (based on the url)")
|
|
for w in ['verified.csv', 'unverified.csv']:
|
|
csvfilepath = os.path.join(instancepath, w)
|
|
print_colors(f"Processing file: {csvfilepath}")
|
|
try:
|
|
csvdf = pd.read_csv(csvfilepath)
|
|
print_colors(f"Removing duplicates in {csvfilepath}")
|
|
csvdf = csvdf.drop_duplicates(subset=['URL'])
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
print_colors(f"Cleaned data:\n{csvdf[['URL']]}")
|
|
except FileNotFoundError:
|
|
print_colors(f"File not found: {csvfilepath}")
|
|
except Exception as e:
|
|
print_colors(f"An error occurred while processing {csvfilepath}: {e}")
|
|
break
|
|
break
|
|
|
|
case 10:
|
|
print_colors("[+] 10) perform sanity checks on all csv files (to mark them as sensitive / or remove the ones that are blacklisted)")
|
|
participantspath = rootpath+'www/participants/'
|
|
for participant in os.listdir(participantspath):
|
|
print_colors(f"Participant: {participant}")
|
|
#read=input("Continue?")
|
|
participantdir= participantspath+participant
|
|
################ BEGIN SANITY CHECKS FOR EACH PARTICIPANTS ##############
|
|
# iterate through the participant's verified.csv and unverified.csv files
|
|
for w in ['verified.csv','unverified.csv']:
|
|
csvfilepath=participantdir+'/'+w
|
|
print_colors(f"{csvfilepath}")
|
|
csvdf = pd.read_csv(csvfilepath)
|
|
rows2delete= [] # it is an empty list at first
|
|
for i,j in csvdf.iterrows():
|
|
row=csvdf.loc[i,:].values.tolist()
|
|
#print_colors(f"{row}")
|
|
|
|
|
|
|
|
################################ SANITY CHECKS ####################################
|
|
### SANITY CHECK 0: make sure that ✔️ and x are replaced with YES/NO, as it changed since v1.0.1 ###
|
|
if csvdf.at[i, 'Status'] == "✔️" or csvdf.at[i, 'Status'] == "YES" :
|
|
csvdf.at[i, 'Status'] = "YES"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
else:
|
|
csvdf.at[i, 'Status'] = "NO"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
|
|
if csvdf.at[i, 'Sensitive'] == "✔️" or csvdf.at[i, 'Sensitive'] == "YES" :
|
|
csvdf.at[i, 'Sensitive'] = "YES"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
else:
|
|
csvdf.at[i, 'Sensitive'] = "NO"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
|
|
### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion###
|
|
if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False:
|
|
if i not in rows2delete:
|
|
print_colors(f"Marking row {i} for deletion, as it has invalid inputs")
|
|
#print_colors(f"{row}")
|
|
print(IsUrlValid(csvdf.at[i, 'Instance']), IsCategoryValid(csvdf.at[i, 'Category']), IsNameValid(csvdf.at[i, 'Name']), IsUrlValid(csvdf.at[i, 'URL']), IsStatusValid(csvdf.at[i, 'Sensitive']), IsDescriptionValid(csvdf.at[i, 'Description']), IsStatusValid(csvdf.at[i, 'Status']), IsScoreValid(csvdf.at[i, 'Score']))
|
|
rows2delete.append(i)
|
|
read=input("Continue?")
|
|
|
|
### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ###
|
|
for k,l in bldf.iterrows():
|
|
blword=bldf.at[k, 'blacklisted-words']
|
|
if any(blword in str(x) for x in row) == True:
|
|
if i not in rows2delete:
|
|
print_colors(f"Marking row {i} for deletion, as it matches with the blacklisted word {blword}")
|
|
rows2delete.append(i)
|
|
#read=input("Continue?")
|
|
### SANITY CHECK 3: Mark all rows that match sensitive words to be sensitive = YES
|
|
for k,l in sedf.iterrows():
|
|
seword=sedf.at[k, 'sensitive-words']
|
|
if any(seword in str(x) for x in row) == True:
|
|
print_colors(f"Marking row {i} as sensitive, as it matches with the sensitive word {seword}")
|
|
csvdf.at[i, 'Sensitive']="YES"
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
#read=input("Continue?")
|
|
|
|
|
|
for i in rows2delete:
|
|
row=csvdf.loc[i,:].values.tolist()
|
|
print_colors(f'[+] REMOVING ROW : {i} {row}')
|
|
csvdf.drop(i, inplace= True)
|
|
csvdf.to_csv(csvfilepath, index=False)
|
|
#read=input("Continue?")
|
|
break
|
|
|
|
case 0:
|
|
print_colors(f"[-] Exiting", bold=True)
|
|
break
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|