option 4 almost completed

scripts/darknet_exploration.py

@@ -1,7 +1,10 @@
-import os, pwd, re, pandas as pd, requests
+import os, pwd, re, pandas as pd, requests, shutil
 #apt install python3-pandas python3-requests python3-socks
 def main():
-
+	proxies = {
+		'http': 'socks5h://127.0.0.1:9050',
+		'https': 'socks5h://127.0.0.1:9050'
+	}
 
 	rootpath='/srv/darknet-onion-webring/'
 	urlpath=pwd.getpwuid(os.getuid()).pw_dir+"/.darknet_participant_url"
@@ -23,11 +26,22 @@ def main():
 				print('[-] Invalid instance name in ~/.darknet_participant_url:', instance)
 				return False
 			instancepath=rootpath+'www/participants/'+instance
+			templatepath=rootpath+'templates/'
 			verifiedcsvfile=instancepath+'/verified.csv'
-			vdf = pd.read_csv(verifiedcsvfile)
 			unverifiedcsvfile=instancepath+'/unverified.csv'
+			# check if instancepath exists, if not then create the directory
+			if not os.path.exists(instancepath):
+				os.makedirs(instancepath)
+			# check if all the required csv files exist in it, otherwise copy them from the templates directory
+			for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv']:
+				filepath=instancepath+'/'+i
+				if not os.path.isfile(filepath):
+					# copy templates/ FILE.CSV to instancepath/ FILE.CSV
+					src=templatepath+i
+					shutil.copyfile(src, filepath)
+			# now that they exist, get vdf and uvdf
+			vdf = pd.read_csv(verifiedcsvfile)
 			uvdf = pd.read_csv(unverifiedcsvfile)
-		    #df = pd.read_csv(csvfile)
 			print("[+] file exists, your Webring URL is", instance)
 			isitvalid = "y"
 		else:
@@ -211,7 +225,8 @@ Managing Wordlists:
 					webring_participant_url = input("What is the onion domain of the new webring participant? (ex: uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion)  ")
 				# check if the directory exists locally or not,
 				participantdir=rootpath+'www/participants/'+webring_participant_url
-				if os.path.isdir(participantdir):
+				if not os.path.isdir(participantdir): # to test on your own instance
+				#if os.path.isdir(participantdir):
 					# if it does, it means that the webring is ALREADY added
 					print("[-] Webring Participant is already listed, skipping.")
 					return False
@@ -231,16 +246,41 @@ Managing Wordlists:
 					w_webcsv=basewurl+'webring-participants.csv'
 					#print(CheckUrl(w_webcsv))
 
+					# verify that their verified.csv csv file exists at basewurl+'verified.csv'
 					if CheckUrl(w_vcsv) is False or CheckUrl(w_uvcsv) is False or CheckUrl(w_blcsv) is False or CheckUrl(w_scsv) is False or CheckUrl(w_webcsv) is False:
 						print("[-] Webring Participant is invalid, exiting.")
 						return False
 					else:
 						print("[+] Webring Participant is valid, adding it.")
-					# verify that their verified.csv csv file exists at basewurl+'verified.csv'
+						# create the directory in www/participants/PARTICIPANTURL/ if it's not there already
+						if not os.path.exists(participantdir):
+							os.makedirs(participantdir)
+						#  then download their csv files at http://URL.onion/participants/URL.onion/{verified.csv,unverified.csv,sensitive.csv,blacklist.csv,webring-participants.csv}
+						# then save the csv file contents into a variable, then write it where it belongs:
+						# for loop with each csv file you want:
+						for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv']:
+							# FOR EACH CSV FILE TO GET:
+							# URL: basewurl / FILE.CSV
+							# PATH: participantdir / FILE.CSV
+							print('[+] DOWNLOADING ',basewurl+i)
+							# download the external csv file and save it into the "text" variable:
+							#response = urllib.request.urlopen(basewurl+i)
+							response = requests.get(basewurl+i, proxies=proxies)
+							#data = response.read()      # a `bytes` object
+							#text = data.decode('utf-8')
+							text = response.text
+							# save the text variable into the destination file:
+							print('[+] SAVING IT INTO ',participantdir+'/'+i)
+							csvfilepath=participantdir+'/'+i
+							with open(csvfilepath, "w") as file:
+								file.write(text)
+							print("[+] file written, let's read it")
+							f = open(csvfilepath,"r")
+							print(f.read())
 							
-					#  then download their csv files at http://URL.onion/participants/URL.onion/{verified.csv,unverified.csv,sensitive.csv,blacklist.csv,webring-participants.csv}
+					
-					#  and remove all of the invalid entries !!!
+						#  TODO and remove all of the invalid entries !!!
-						# TODO if OK then add it to the webring-participants.csv file 
+						# TODO if OK then add it to the webring-participants.csv file and also in the templates csv file !!! 
 					#######################################################################
 					#newrow=[instance,category,name,url,sensi,desc,'','']
 					#print("[+] NEWROW=",newrow)

templates/webring-participants.csv

@@ -1,2 +1,2 @@
 Name,URL,Description,Trusted,Status,Score
-Nowhere,http://uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,"Darknet Webring Onion Participant",✔️,✔️,100.0
+Nowhere,http://uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,"Darknet Webring Onion Participant",,✔️,100.0