v1.0.1 release candidate 1

.gitignore

@@ -1,3 +1,3 @@
 .git
-www/participants/**
+www/participants/*/*.csv
 

scripts/uptimechecker.py

@@ -1,4 +1,4 @@
-import os,pwd,re
+import os,re,pwd
 import csv
 import requests
 import json
@@ -13,6 +13,7 @@ def main():
 	# TODO get the instance name and exit if its not there
 	rootpath='/srv/darknet-lantern/'
 	urlpath=pwd.getpwuid(os.getuid()).pw_dir+"/.darknet_participant_url"
+
 	#print(urlpath)
 
 
@@ -24,7 +25,6 @@ def main():
 			with open(urlpath) as f:
 				instance = f.read().rstrip()
 			# check if the instance URL domain is valid
-			#print(urlpath,instance)
 			if IsOnionValid(instance):
 				print("[+] Instance Name:",instance,IsOnionValid(instance))
 				isitvalid="y"
@@ -35,7 +35,6 @@ def main():
 			print("[-] Instance path doesn't exist yet, run darknet_exploration.py to set it up" )
 			return False
 	
-	#i=input("continue?")	
 	proxies = {
 		'http': 'socks5h://127.0.0.1:9050',
 		'https': 'socks5h://127.0.0.1:9050'
@@ -72,38 +71,37 @@ def main():
 				print('[+]',url,status)
 				if status != 502:
 					print(url,"✔️")
-					df.at[i,"Status"]="✔️"
+					df.at[i,"Status"]="YES"
 					#if uptime <100 do +1 to the value
 					if df.at[i,"Score"] < 100:
 						df.at[i,"Score"] = df.at[i,"Score"] + 1
 				else:
 					print(url,"❌")
-					df.at[i,"Status"]="❌"
+					df.at[i,"Status"]="NO"
 					#if uptime >0  do -1 to the value
 					if df.at[i,"Score"] > 0:
 						df.at[i,"Score"] = df.at[i,"Score"] - 1
 			except requests.ConnectionError as e:
 				#print(e)
 				print(url,"❌")
-				df.at[i,"Status"]="❌"
+				df.at[i,"Status"]="NO"
 				#if uptime >0  do -1 to the value
 				if df.at[i,"Score"] > 0:
 					df.at[i,"Score"] = df.at[i,"Score"] - 1
 			except requests.exceptions.ReadTimeout as e:
 				#print(e)
 				print(url,"❌")
-				df.at[i,"Status"]="❌"
+				df.at[i,"Status"]="NO"
 				#if uptime >0  do -1 to the value
 				if df.at[i,"Score"] > 0:
 					df.at[i,"Score"] = df.at[i,"Score"] - 1
 
+			df2 = df.sort_values(by=["Score"], ascending=False)
 			#sort by category if you are verified/unverified.csv
 			if csvfilename in csvfiles2sortcat:
-				df2 = df.sort_values(by=["Category","Name"], ascending=[True,True])
-			else:
-				df2 = df.sort_values(by="Score", ascending=False)
+				df2 = df.sort_values(by=["Category"], ascending=True)
+			#print(df2)
 			df2.to_csv(csvfile, index=False)
-		#print(df2)
 
 
 def IsUrlValid(url:str)->bool:
@@ -167,19 +165,14 @@ def IsOnionValid(url: str)-> bool:
                 if len(url.split('.')) > 3:
                     n_subdomians = len(url.split('.'))
                     # Checks if there is more than 1 subdomain. "subdomain.url.onion" only
-                    #print(f"This domain have more than one subdomain. There are {n_subdomians - 1} subdomains")
                     return False
                 else:
                     if len(url) < 62:
-                        #print("Domain length is less than 62.")
                         return False
                     return True
             elif pattern.fullmatch(url) is None:
-                #print("Domain contains invalid character.")
-                #print(url)
                 return False
             else:
-                #print("Domain not valid")
                 return False
     except Exception as e:
         print(f"Error: {e}")

scripts/utils.py

@@ -1,158 +1,120 @@
-import os, pwd, re, pandas as pd, requests, shutil
+import re
+import requests
 from PIL import Image
-import urllib
-import socks, socket, glob
 
 PURPLE =  '\033[35;40m'
 BOLD_PURPLE = '\033[35;40;1m'
-ORANGE = '\033[33;40;1m'
 RED = '\033[31;40m'
 BOLD_RED = '\033[31;40;1m'
 RESET = '\033[m'
 
 
+
 #### Checking Functions to validate that links are legit ####
 
 def CheckUrl(url):
-	"""
-	Checks if URL is actually reachable via Tor
-	"""
-	proxies = {
-		'http': 'socks5h://127.0.0.1:9050',
-		'https': 'socks5h://127.0.0.1:9050'
-	}
-	try:
-		status = requests.get(url,proxies=proxies, timeout=5).status_code
-		print('[+]',url,status)
-		if status != 502:
-			#print(url,"✔️")
-			return True
-		else:
-			#print(url,"❌")
-			return False
-	except requests.ConnectionError as e:
-		#print(url,"❌")
-		return False
-	except requests.exceptions.ReadTimeout as e:
-		#print(url,"❌")
-		return False
+        """
+        Checks if URL is actually reachable via Tor
+        """
+        proxies = {
+                'http': 'socks5h://127.0.0.1:9050',
+                'https': 'socks5h://127.0.0.1:9050'
+        }
+        try:
+                status = requests.get(url,proxies=proxies, timeout=5).status_code
+                if status != 502:
+                        return True
+                else:
+                        return False
+        except requests.ConnectionError as e:
+                return False
+        except requests.exceptions.ReadTimeout as e:
+                return False
+
 
 #### PROTECTIONS AGAINST MALICIOUS CSV INPUTS ####
 
 def IsBannerValid(path: str) -> bool:
-	"""
-	Checks if the banner.png file has the correct dimensions (240x60)
-	"""
-    #print('[+] checking image size')
-	try:
-		im = Image.open(path)
-	except Exception as e:
-		return False
-    #im = Image.open("favicon.png")
-	width, height = im.size
-	#print('width =',width, 'height=',height)
-	if width != 240 or height != 60:
-		#print('[-] Banner doesnt have the correct size (240x60)')
-		return False
-	else:
-		#print('[+] Banner has the correct size (240x60)')
-		return True
+        """
+        Checks if the banner.png file has the correct dimensions (240x60)
+        """
+        try:
+                im = Image.open(path)
+        except Exception as e:
+                print("ERROR, EXCEPTION")
+                return False
+        width, height = im.size
+        if width != 240 or height != 60:
+                print("INVALID BANNER DIMENSIONS, HEIGHT=",height," WIDTH=",width)
+                return False
+        
+        return True
 
 
 def IsOnionValid(url: str)-> bool:
     """
     Checks if the domain(param) is a valid onion domain and return True else False.
     """
-        # check if the characters are only [a-zA-Z0-9.] with maximum 128 chars max?
-                # check that it is only url.onion or subdomain.url.onion,
-                        #  if OK return True
-        #if not : return False
     try:
-        pattern = re.compile("^[A-Za-z0-9.]+(\.onion)?$")
+        pattern = re.compile("^[A-Za-z0-9.]+(.onion)?$")
         url = url.strip().removesuffix('/')
         if url.startswith('http://'):
-            #print('URL starts with http')
-            # Removes the http://
             domain = url.split('/')[2]
             if pattern.fullmatch(domain) is not None:
                 if len(domain.split('.')) > 3:
-                    n_subdomians = len(domain.split('.'))
-                    # Checks if there is more than 1 subdomain. "subdomain.url.onion" only
-                    #print(f"This domain have more than one subdomain. There are {n_subdomians} subdomains")
                     return False
                 else:
                     if len(domain) < 62:
-                        #print("Domain length is less than 62.")
                         return False
                     return True
             elif pattern.fullmatch(domain) is None:
-                #print("Domain contains invalid character.")
-                #print(domain)
                 return False
             else:
-                #print("Domain not valid")
                 return False
         else:
-			#TODO : edit the url to make sure it has http:// at the beginning, in case if it's missing? (problem is that it only returns true or false)
-            #print("URL doesn't start http")
+                        #TODO : edit the url to make sure it has http:// at the beginning, in case if it's missing? (problem is that it only returns true or false)
             if pattern.fullmatch(url) is not None:
                 if len(url.split('.')) > 3:
-                    n_subdomians = len(url.split('.'))
-                    # Checks if there is more than 1 subdomain. "subdomain.url.onion" only
-                    #print(f"This domain have more than one subdomain. There are {n_subdomians - 1} subdomains")
                     return False
                 else:
                     if len(url) < 62:
-                        #print("Domain length is less than 62.")
                         return False
                     return True
             elif pattern.fullmatch(url) is None:
-                #print("Domain contains invalid character.")
-                #print(url)
                 return False
             else:
-                #print("Domain not valid")
                 return False
     except Exception as e:
-        print(f"Error: {e}")
+        return False
+
 
 def IsUrlValid(url:str)->bool:
         """
         Check if url is valid both dark net end clearnet.
         """
-        # check if the characters are only [a-zA-Z0-9.:/] with maximum 128 chars max?
-                # check that it is only http(s)://wordA.wordB or http(s)://WordC.WordB.WordC, (onion or not), clearnet is fine too (double check if those are fine!)
-                        #  if OK return True
-        #if not : return False
         pattern  = re.compile("^[A-Za-z0-9:/.-]+$")
         url = str(url)
         if len(url) < 4:
-                #print("Status: Got more than one character or nothing.")
                 return False
         if url.endswith('.onion'):
                 return IsOnionValid(url)
         else:
                 if not url.__contains__('.'):
-                        #print("No (DOT) in clearnet url")
                         return False
                 if pattern.fullmatch(url) is None:
-                        #print('Url contains invalid chars')
                         return False
                 return True
 
+
 def IsStatusValid(status: str)-> bool:
         """
-        Checks if status contains only [v,x,❌,✔️]. Verbose only if False is returned
+        Checks if status contains only ['YES','NO']. Verbose only if False is returned
         """
-        pattern = ['y','n','✔️','❌','','nan']
+        pattern = ['YES','NO','✔️','❌','']
+        #pattern = ['YES','NO']
         status = str(status)
         status.strip()
-        #print('[+] STATUS = ',status.splitlines())
-        if len(status) > 4:
-                #print("Status: Got more than one character or nothing.")
-                return False
-        elif (status not in pattern):
-                #print("Status: Got an invalid character it must be either y, n, ✔️, or ❌ ")
+        if (status not in pattern):
                 return False
 
         return True
@@ -162,24 +124,15 @@ def IsScoreValid(score:str)->bool:
         """
         Check the Score is only "^[0-9.,]+$" with 8 max chars.
         """
-        # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max
-                #(careful with the ' and , make sure you test if it fucks the csv up or else)
-                #  if OK return True
-        #if not : return False
         pattern = re.compile("^[0-9.,]+$")
         score = str(score)
         score.strip()
-        #pattern = ['','nan']
         if score in ['','nan']:
-            #Score can be empty when initially added
             return True
         if pattern.fullmatch(score) is None:
-                # empty description is fine as it's optional
                 return False
         elif len(score) > 8:
-                #print("score is greater than 8 chars")
                 return False
-        # empty score is fine
         return True
 
 
@@ -187,12 +140,7 @@ def IsDescriptionValid(desc:str)->bool:
         """
         Check the categories are only [a-zA-Z0-9.' ] with 256 max chars.
         """
-        # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max
-                #(careful with the ' and , make sure you test if it fucks the csv up or else)
-                #  if OK return True
-        #if not : return False
         if desc == "":
-                # empty description is fine as it's optional
                 return True
         pattern = re.compile("^[A-Za-z0-9-.,' \"]+$")
         desc = str(desc)
@@ -202,7 +150,6 @@ def IsDescriptionValid(desc:str)->bool:
         if desc == "DEFAULT":
                 return False
         elif len(desc) > 256:
-                #print("desc is greater than 256 chars")
                 return False
         return True
 
@@ -210,18 +157,12 @@ def IsCategoryValid(categories: list)-> bool:
         """
         Check the categories are only [a-zA-Z0-9 ] with 64 max chars.
         """
-        # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max
-                #(careful with the ' and , make sure you test if it fucks the csv up or else)
-                #  if OK return True
-        #if not : return False
         pattern = re.compile("^[A-Za-z0-9 ]+$")
         for category in categories:
                 category.strip()
                 if pattern.fullmatch(category) is None:
-                        #print('Got an empty list or invalid chars')
                         return False
                 elif len(category) > 64:
-                        #print('Category is too long')
                         return False
                 else:
                        return True
@@ -230,41 +171,31 @@ def IsNameValid(name: str)->bool:
         """
         Check the parameter name only contains [a-zA-Z0-9 ] and is 64 chars long.
         """
-        # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max
-                #(careful with the ' and , make sure you test if it fucks the csv up or else)
-                #  if OK return True
-        #if not : return False
+        try:
+                name = str(name)
+        except Exception as e:
+                return False
         pattern = re.compile("^[A-Za-z0-9 ]+$")
         name = name.strip()
         if (pattern.fullmatch(name) is None):
-                #print("Got an invalid character or nothing")
                 return False
         elif len(name) > 64:
-                #print(f'Got a name length greater than 64. {len(name)}')
                 return False
         return True
 
 
-
-#def print_colors(s:str, bold=False, is_error = False, default=False):
-def print_colors(*args, bold=False, is_error=False, default=False, highlight=False):
-    """
-    Helper function to print with colors
-    """
-    for s in args:
+def print_colors(s:str=' ', bold:bool=False, is_error:bool = False, default:bool=False):
+        """
+        Helper function to print with colors
+        """
         if is_error:
-                print(f"{RED}{s}{RESET}",end='')
-        elif highlight:
-                print(f"{ORANGE}{s}{RESET}",end='')
+                print(f"{RED}{s}{RESET}")
         elif bold:
-                print(f"{BOLD_PURPLE}{s}{RESET}",end='')
+                print(f"{BOLD_PURPLE}{s}{RESET}")
         elif is_error and bold:
-                print(f"{BOLD_RED}{s}{RESET}",end='')
+                print(f"{BOLD_RED}{s}{RESET}")
         elif default:
-                print(f'{s}',end='')
+                print(f'{s}')
         else:
-                print(f"{PURPLE}{s}{RESET}",end='')
-        if s is args[-1]:
-                print()
-			
+                print(f"{PURPLE}{s}{RESET}")
         

updaterepo.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+#warning, you need to have the .gitignore intact to ignore www/participants/*/**
+
+git rm -r --cached .
+git add .
+git commit 
+torsocks git push

www/.known_participants

@@ -0,0 +1,3 @@
+lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion
+lantern.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion
+zhd7yf675dav6njgc7yjwke2u5cq7d5qim2s7xwa2ukxfzubrguqmzyd.onion

www/header.php

@@ -53,7 +53,7 @@ if (!preg_match("~^(?:f|ht)tps?://~i", $data[3])) {
 
 		//if ((($sensitive == 1) and ($data[4] == "✔️"))  or (($sensitive == 0) and ($data[4] != "✔️"))  ){ 
 		// ONLY display links if (sensitive equals to 1 and sensitiveCOLUMN equals to V) OR (sensitive equals to 0 and sensitiveCOLUMN is NOT equal to V)
-		if (($data[4] != "✔️") or (($sensitive == 1) and ($data[4] == "✔️"))){ 
+		if (($data[4] != "YES") or (($sensitive == 1) and ($data[4] == "YES"))){ 
 		$rowcount++;
 
 
@@ -67,7 +67,7 @@ if (!preg_match("~^(?:f|ht)tps?://~i", $data[3])) {
 		}
 
                     echo "<td>" ; // begin the table cell  
-					if($data[4] == "✔️"){
+					if($data[4] == "YES"){
 						echo '<a class="sensitivelink" href="'; // begin a href
 					}else{
 						echo '<a href="'; // begin a href
@@ -76,8 +76,13 @@ if (!preg_match("~^(?:f|ht)tps?://~i", $data[3])) {
                     echo  $urllink . '"> '; // display the link
                     echo  $data[2]  . ' </a></td><td class="description">'; // display the link title and close the a href  and first cell, open the second cell
                     echo  $data[5] . " </td><td>"; // OPTIONAL: display the description column
-                    echo  $data[7] . " </td><td>"; // display the status and close the second cell, open the third cell
-                    echo  $data[6] . " </td> \n"; // display the score and close the third cell
+                    echo  $data[7] . " </td><td>"; // display the score and close the second cell, open the third cell
+					if($data[6] == "YES"){
+						echo "✔️" ;
+					}else{
+						echo "❌" ;
+					}
+                    echo  " </td> \n"; // display the status and close the third cell
             }
             //if ($c == 2){
             //}
@@ -169,7 +174,7 @@ if (($handle = fopen($csvfile, "r")) !== FALSE) {
 		}
         }
     }
-    echo "</p>";
+    echo '<a class="sensitivelink" href="index.php?query=.&sensitive=1">Display All Links</a> |</p>';
     fclose($handle);
 }
 //echo "<p>" . $resultcount . " Result(s) found.</p>";