option 4 wip

scripts/darknet_exploration.py

@@ -1,6 +1,8 @@
-import os, pwd, re, pandas as pd
+import os, pwd, re, pandas as pd, requests
+#apt install python3-pandas python3-requests python3-socks
 def main():
 
+
 	rootpath='/srv/darknet-onion-webring/'
 	urlpath=pwd.getpwuid(os.getuid()).pw_dir+"/.darknet_participant_url"
 	#print(urlpath)
@@ -33,7 +35,7 @@ def main():
 			# and ask for the instance URL domain 
 			instance = input("What is your Instance domain ? (ex: uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion):")
 			instancepath=rootpath+'www/participants/'+instance
-			# TODO check if the instance URL domain is valid
+			# check if the instance URL domain is valid
 			if IsOnionValid(instance):
 				print("[+] Instance Name: ",instance,IsUrlValid(instance))
 			else:
@@ -57,28 +59,34 @@ def main():
 	while True:	
 		print("[+] Welcome to the Darknet Onion Webring, where you are exploring the Darknet and helping others do the same.")
 		print("""
-Websites:
+Managing Websites:
  1) Add a new Website entry (into unverified.csv) 
  2) Trust a Website entry (move an entry from unverified to verified.csv) 
  3) Untrust a Website entry (move an entry from unverified to verified.csv) 
 
-Webring Participants:
+Managing Webring Participants:
  4) Add a new webring participant (and download their files into their directory (without trusting them yet!))
  5) Trust a webring participant (Potentially dangerous)
  6) Untrust a webring participant
  7) Remove a webring participant
 
-Wordlists:
+Managing Wordlists:
- 8) Add/Remove words in the sensitive list (ex: drug)
+ 8) Add/Remove words or links in the sensitive list (ex: drug)
- 9) Add/Remove words in the blacklist (ex: porn)
+ 9) Add/Remove words or links in the blacklist (ex: porn)
  0) Exit
 		""")
-		option = input("Select Option? (0-6)")
+		option = input("Select Option? (0-6): ")
 		print(option)
 		match option:
 
 
 ########## MANAGING WEBSITE ENTRIES #################
+#Websites:
+# 1) Add a new Website entry (into unverified.csv) 
+# 2) Trust a Website entry (move an entry from unverified to verified.csv) 
+# 3) Untrust a Website entry (move an entry from unverified to verified.csv) 
+#####################################################
+
 			case "1":
 				print("\n[+] Add a new Website entry (into unverified.csv)")
 				name=''
@@ -121,7 +129,7 @@ Wordlists:
 				# search for a word 
 				name=''
 				while(IsNameValid(name) is not True):
-				    name = input("What is the Website name you want to verify ? (ex: Nowhere)")
+				    name = input("What is the Website name you want to trust ? (ex: Nowhere)")
 				filter_uvdf = uvdf[uvdf.Name.str.contains(name)]
 				# and display only the matching entries in unverified.csv in an array format (display it in CLI). 
 				print(filter_uvdf[['Name','URL']])	
@@ -153,8 +161,40 @@ Wordlists:
 					print("[+] Link is now moved to verified.csv!")
 
 			case "3":
-				# TODO
 				print("[+] Untrust a Website entry (move an entry from verified to unverified.csv)")
+				# search for a word 
+				name=''
+				while(IsNameValid(name) is not True):
+				    name = input("What is the Website name you want to untrust ? (ex: BreachForums)")
+				filter_vdf = vdf[vdf.Name.str.contains(name)]
+				# and display only the matching entries in unverified.csv in an array format (display it in CLI). 
+				print(filter_vdf[['Name','URL']])	
+				# check if there are no results, dont proceed if there are none!
+				if filter_vdf.size == 0:
+					print("ERROR no results, skipping.")
+				else:
+					# Each of the rows has an index,
+					index=-1
+					while (index not in filter_vdf.index):
+						# prompt the user to ask for with row they want to move to unverified.csv
+						index = int(input("What is the index of the entry that you want to move to unverified.csv ? (ex: 3) "))
+					#  once selected, it must be able to SAVE and print that row:
+					print(vdf.iloc[index].values)
+					newrow=vdf.iloc[index].values
+				
+
+					#  append it into unverified.csv
+					uvdf.loc[-1] = newrow  # adding a row
+					uvdf.index = uvdf.index + 1  # shifting index
+					uvdf = uvdf.sort_index()  # sorting by index
+					uvdf.to_csv(unverifiedcsvfile, index=False)
+					print("[+] New row added to unverified.csv!")
+
+				
+					#  remove it from verified.csv
+					vdf.drop(index, inplace= True)
+					vdf.to_csv(verifiedcsvfile, index=False)
+					print("[+] Link is now moved to unverified.csv!")
 
 
 ####### MANAGING WEBRING PARTICIPANTS ###########
@@ -165,10 +205,51 @@ Wordlists:
 #####################################################
 			case "4":
 				print("[+] Add a new webring participant (and download their files into their directory (without trusting them yet!))")
-				#  TODO ask for the url to the other webring participant
+				webring_participant_url = '' 
-				webring_participant_url = input("What is URL of the new webring participant? (ex: https://torproject.org or http://2gzyxa5ihm7nsggfxnu52rck2vv4rvmdlkiu3zzui5du4xyclen53wid.onion")
+				while(IsOnionValid(webring_participant_url) is not True):			
-				#  TODO check if the (onion only) url is valid or not: at http://URL.onion/participants/URL.onion/{verified.csv,unverified.csv,sensitive.csv,blacklist.csv,webring-participants.csv}
+					# ask for the url to the other webring participant and check if the (onion only) url is valid or not: 
+					webring_participant_url = input("What is the onion domain of the new webring participant? (ex: uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion)  ")
+				# check if the directory exists locally or not,
+				participantdir=rootpath+'www/participants/'+webring_participant_url
+				if os.path.isdir(participantdir):
+					# if it does, it means that the webring is ALREADY added
+					print("[-] Webring Participant is already listed, skipping.")
+					return False
+				else:
+					# if not, then proceed:
+					#  print the URL to the csv files at http://URL.onion/participants/URL.onion/{verified.csv,unverified.csv,sensitive.csv,blacklist.csv,webring-participants.csv}
+					basewurl='http://'+webring_participant_url+'/participants/'+webring_participant_url+'/'
+					print(basewurl)
+					print('[+] Checking if all of the required csv files exists for new webring participant ',webring_participant_url, ": ")
+					w_vcsv=basewurl+'verified.csv'
+					w_uvcsv=basewurl+'unverified.csv'
+					#print(CheckUrl(w_uvcsv))
+					w_blcsv=basewurl+'blacklist.csv'
+					#print(CheckUrl(w_blcsv))
+					w_scsv=basewurl+'sensitive.csv'
+					#print(CheckUrl(w_scsv))
+					w_webcsv=basewurl+'webring-participants.csv'
+					#print(CheckUrl(w_webcsv))
+
+					if CheckUrl(w_vcsv) is False or CheckUrl(w_uvcsv) is False or CheckUrl(w_blcsv) is False or CheckUrl(w_scsv) is False or CheckUrl(w_webcsv) is False:
+						print("[-] Webring Participant is invalid, exiting.")
+						return False
+					else:
+						print("[+] Webring Participant is valid, adding it.")
+					# verify that their verified.csv csv file exists at basewurl+'verified.csv'
+					
+					#  then download their csv files at http://URL.onion/participants/URL.onion/{verified.csv,unverified.csv,sensitive.csv,blacklist.csv,webring-participants.csv}
+					#  and remove all of the invalid entries !!!
 						# TODO if OK then add it to the webring-participants.csv file 
+					#######################################################################
+					#newrow=[instance,category,name,url,sensi,desc,'','']
+					#print("[+] NEWROW=",newrow)
+					#uvdf.loc[-1] = newrow  # adding a row
+					#uvdf.index = uvdf.index + 1  # shifting index
+					#uvdf = uvdf.sort_index()  # sorting by index
+					#print("[+] New row added! now writing the csv file:")
+					#uvdf.to_csv(unverifiedcsvfile, index=False)
+
 			
 			case "5":
 				print("[+] Trust a webring participant (Potentially dangerous)")
@@ -237,8 +318,33 @@ Wordlists:
 				return True
 
 
-#### PROTECTIONS AGAINST MALICIOUS CSV INPUTS ####
+#### Checking Functions to validate that links are legit ####
 
+def CheckUrl(url):
+	"""
+	Checks if URL is actually reachable via Tor
+	"""
+	proxies = {
+		'http': 'socks5h://127.0.0.1:9050',
+		'https': 'socks5h://127.0.0.1:9050'
+	}
+	try:
+		status = requests.get(url,proxies=proxies, timeout=5).status_code
+		print('[+]',url,status)
+		if status != 502:
+			print(url,"✔️")
+			return True
+		else:
+			print(url,"❌")
+			return False
+	except requests.ConnectionError as e:
+		print(url,"❌")
+		return False
+	except requests.exceptions.ReadTimeout as e:
+		print(url,"❌")
+		return False
+
+#### PROTECTIONS AGAINST MALICIOUS CSV INPUTS ####
 
 def IsOnionValid(url: str)-> bool:
     """