wrote the remaining todos

scripts/darknet_exploration.py

@@ -35,7 +35,7 @@ def main():
 			if not os.path.exists(instancepath):
 				os.makedirs(instancepath)
 			# check if all the required csv files exist in it, otherwise copy them from the templates directory
-			# NOTE : the templates files are EMPTY by default, this is because i want each peer to manually review lists of links, and links themselves manually, do not allow malicious links to slip through without intentional edits from the peer themselves.
+			# NOTE : the templates files are EMPTY by default, this is because i want each peer to manually review lists of links, and links themselves manually, this is to avoid allowing malicious links to slip through without intentional edits from the peer themselves.
 			for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv']:
 				filepath=instancepath+'/'+i
 				if not os.path.isfile(filepath):
@@ -92,6 +92,11 @@ Managing Webring Participants:
 Managing Wordlists:
  8) Add/Remove words or links in the sensitive list (ex: drug)
  9) Add/Remove words or links in the blacklist (ex: porn)
+
+Maintenance:
+10) remove the duplicate URLs
+11) perform sanity checks on all csv files (to mark them as sensitive / or remove the ones that are blacklisted) 
+
  0) Exit
 		""")
 		option = input("Select Option? (0-6): ")
@@ -257,13 +262,13 @@ Managing Wordlists:
 						return False
 					else:
 						print("[+] Webring Participant is valid, adding it.")
-						# TODO if OK then add it to your own webring-participants.csv
 						name=''
 						while(IsNameValid(name) is not True):
 						    name = input("What is the Webring instance name ? ")
 						desc='DEFAULT'
 						while(IsDescriptionValid(desc) is not True):
 							desc=input("Description for the webring participant ? (Optional)")
+						# if OK then add it to your own webring-participants.csv
 						trusted=''
 						status=''
 						score=''
@@ -364,19 +369,30 @@ Managing Wordlists:
 			case "5":
 				print("[+] Trust a webring participant (Potentially dangerous)")
 				#  TODO print a warning to tell the user that you're about to trust all of the links that are trusted by another peer, if that peer is malicious they may start trusting links that you don't trust! proceed? (answer is y otherwise skip)
-				#  TODO list the existing webring participants from your own webring-participants.csv and 
-				#  TODO ask the user to pick the index (0-9?) of the instance to trust
-				#  TODO if index is valid, then mark the instance as trusted in webring-participants.csv
+				choice=''
+				choice=input("You're about to trust another peer, this means that you're going to automatically trust all of the links they have in their verified.csv file! If this is a malicious peer, you're about to potentially trust malicious links, it is potentially risky! Do you want to continue ? (y/n)")
+				if choice == "y":
+					# TODO list all of the webring participants in webring-participants.csv
+					print("Please choose a webring participant to trust:")	
+					# TODO make the user choose a valid index	
+					# TODO and write V into the "Trusted" column
+				else:
+					print("Exiting.")	
 				searchterm = input("What is the index of the entry that you want to trust ?")
 			case "6":
 				# TODO
 				print("[+] Untrust a webring participant (safer)")
+				print("Please choose a webring participant to untrust:")	
+				# TODO make the user choose a valid index	
+				# TODO and write "" into the "Trusted" column
 			case "7":
 				# TODO
 				print("[+] Remove a webring participant (in case of abuses)")
+				print("Please choose a webring participant to blacklist:")	
 				# TODO add it to blacklist.csv 
+				# TODO remove all of the entries that came from that participant (drop the lines in verified+unverified.csv that have that instance in the instance column) 
 				# TODO remove it from your own webring-participants.csv
-				# TODO remove it from the template webring-participants.csv
+				# TODO remove remove the directory in www/participants/INSTANCENAME
 
 
 
@@ -423,12 +439,51 @@ Managing Wordlists:
 				
 			case "9":
 				print("[+] Add/Remove words in the blacklist list (ex: porn)")
-				# TODO  copy option 5
-				# TODO  print("do you want to 1) add words  or 2) remove words ?")
+				option="0"
+	
+				done = False
+				while(done == False):
+					while option != "1" and option != "2" and option != "exit":
+						option=input("do you want to 1) add words  or 2) remove words ? (type exit to exit)")
 					#  TODO display the contents of blacklist.csv file
+					if option == 1: 
+						word=input("which word do you want to add? (write 0 to exit")
+						if word == "0":
+							done = True
+							#True to get out of the while loop
+						else:
+							print("checking if word is valid")
+							# TODO check if word is valid in a while loop (dont check if 
+							#  TODO if invalid! remove word at index 
+					else:
+						index=input("which word do you want to remove? (index 0 to (max index) (write exit to exit)")
+						if index == "exit": 
+							done = True
+							#True to get out of the while loop
+						else:
+							print("checking if index is valid")
+							# TODO check if index is valid or not
+								# TODO  if valid! remove word at index 
+								# TODO  if invalid! just pass to ask for another word
 			
 			# TODO CASE 10 : cleanup all duplicates in unverified + verified.csv, based on the url (check if each url appears more than once, and if they do, remove them + write to csv file)
-	
+			case "10":
+				print("[+] Cleaning up all duplicates in your own unverified + verified.csv (based on the url)")
+				# TODO for unverified.csv, and verified.csv
+					# TODO iterate through each row of the csv file
+						# TODO get the index of that row (save it as indextocheck), and the url in that row
+							# TODO for unverified.csv, and verified.csv	
+								# check if that URL appears more than once in both unverified.csv and verified.csv,
+									# ignore it if the index is "indextocheck" and if the index is already listed in rows2delete
+									# else: add the index to "rows2delete"
+					# go drop the rows by their index listed in "rows2delete"
+			case "11":
+				print("11) perform sanity checks on all csv files (to mark them as sensitive / or remove the ones that are blacklisted)")
+				# TODO find the list of all csv files (in www/participants/*/*.csv) (templates should remain empty by default)
+					# copy what was done in option 4, to :
+						# delete the ones that have invalid entries
+						# mark the sensitive rows as sensitive
+						# delete the rows that match with blacklisted words
 			case _:
 				print("[-] Exiting")
 				return True
@@ -496,6 +551,7 @@ def IsOnionValid(url: str)-> bool:
                 #print("Domain not valid")
                 return False
         else:
+			#TODO : edit the url to make sure it has http:// at the beginning, in case if it's missing? (problem is that it only returns true or false)
             #print("URL doesn't start http")
             if pattern.fullmatch(url) is not None:
                 if len(url.split('.')) > 3: