diff --git a/README.md b/README.md index 871a28c..7f31699 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ DONE: -py : option 6) Trust/Untrust/Blacklist a webring participant -php : make a search engine prompt that only accepts [a-zA-Z.://], it must refuse every other character +-py : fix uptimecheck.py to match the new csv format DOING: -php : if valid make it filter your own verified.csv and unverified.csv files @@ -12,10 +13,6 @@ DOING: -py : option 8) Add/Remove words in the blacklist (assigned to anon) TODO: --py : fix uptimecheck.py to match the new csv format - -# secondary features: --php : interactive search engine ? it displays as you type it ? -py : option 9) cleanup all duplicates in your own unverified.csv and verified.csv -py : option 10) perform sanity checks on all csv files (to mark them as sensitive or remove the ones that are blacklisted) diff --git a/scripts/uptimecheckernew.py b/scripts/uptimecheckernew.py new file mode 100644 index 0000000..07e96d2 --- /dev/null +++ b/scripts/uptimecheckernew.py @@ -0,0 +1,185 @@ +import os,pwd,re +import csv +import requests +import json +import pandas as pd +import glob + +#apt install python3-pandas python3-requests python3-socks + +def main(): + print('[+] ONION UPTIME CHECKER') + + # TODO get the instance name and exit if its not there + rootpath='/srv/darknet-onion-webring/' + urlpath=pwd.getpwuid(os.getuid()).pw_dir+"/.darknet_participant_url" + #print(urlpath) + + + # check if ~/.darknet_participant_url exists, + # if exists, instance= the content of ~/.darknet_participant_url (which is the url: such as uptime.nowherejez...onion) + isitvalid="n" + while isitvalid != "y": + if os.path.isfile(urlpath): + with open(urlpath) as f: + instance = f.read().rstrip() + # check if the instance URL domain is valid + #print(urlpath,instance) + if IsOnionValid(instance): + print("[+] Instance Name:",instance,IsOnionValid(instance)) + isitvalid="y" + else: + print('[-] Invalid instance name in ~/.darknet_participant_url:', instance) + return False + else: + print("[-] Instance path doesn't exist yet, run darknet_exploration.py to set it up" ) + return False + + #i=input("continue?") + proxies = { + 'http': 'socks5h://127.0.0.1:9050', + 'https': 'socks5h://127.0.0.1:9050' + } + + instancepath=rootpath+'www/participants/'+instance+'/' + csvfiles2check=['verified.csv','unverified.csv','webring-participants.csv'] + + #for csvfile in glob.glob("/srv/darknet-onion-webring/www/links/*.csv"): + for csvfilename in csvfiles2check: + csvfile = instancepath+csvfilename + print('[+] Reading the CSV File:', csvfile) + + df = pd.read_csv(csvfile) + print(df[['Name','URL']]) + print('[+] Checking if each .onion link is reachable:') + #for i in range(df.index.stop): + for i in df.index: + print("[+] Editing the uptime score") + #if empty, set to 100 + if pd.isnull(df.at[i,"Score"]): + df.at[i,"Score"] = 100 + + print(i) + #print(df.at[i,"URL"]) + url=df.at[i,"URL"] + try: + index1 = url.find("http://") + index2 = url.find("https://") + if index1 == -1 and index2 == -1: + url = "http://"+url + status = requests.get(url,proxies=proxies, timeout=5).status_code + print('[+]',url,status) + if status != 502: + print(url,"✔️") + df.at[i,"Status"]="✔️" + #if uptime <100 do +1 to the value + if df.at[i,"Score"] < 100: + df.at[i,"Score"] = df.at[i,"Score"] + 1 + else: + print(url,"❌") + df.at[i,"Status"]="❌" + #if uptime >0 do -1 to the value + if df.at[i,"Score"] > 0: + df.at[i,"Score"] = df.at[i,"Score"] - 1 + except requests.ConnectionError as e: + #print(e) + print(url,"❌") + df.at[i,"Status"]="❌" + #if uptime >0 do -1 to the value + if df.at[i,"Score"] > 0: + df.at[i,"Score"] = df.at[i,"Score"] - 1 + except requests.exceptions.ReadTimeout as e: + #print(e) + print(url,"❌") + df.at[i,"Status"]="❌" + #if uptime >0 do -1 to the value + if df.at[i,"Score"] > 0: + df.at[i,"Score"] = df.at[i,"Score"] - 1 + + df2 = df.sort_values(by=["Score"], ascending=False) + #print(df2) + df2.to_csv(csvfile, index=False) + + +def IsUrlValid(url:str)->bool: + """ + Check if url is valid both dark net end clearnet. + """ + # check if the characters are only [a-zA-Z0-9.:/] with maximum 128 chars max? + # check that it is only http(s)://wordA.wordB or http(s)://WordC.WordB.WordC, (onion or not), clearnet is fine too (double check if those are fine!) + # if OK return True + #if not : return False + pattern = re.compile("^[A-Za-z0-9:/.]+$") + url = str(url) + if url.endswith('.onion'): + return IsOnionValid(url) + else: + if not url.__contains__('.'): + #print("No (DOT) in clearnet url") + return False + if pattern.fullmatch(url) is None: + #print('Url contains invalid chars') + return False + return True + +def IsOnionValid(url: str)-> bool: + """ + Checks if the domain(param) is a valid onion domain and return True else False. + """ + # check if the characters are only [a-zA-Z0-9.] with maximum 128 chars max? + # check that it is only url.onion or subdomain.url.onion, + # if OK return True + #if not : return False + try: + pattern = re.compile("^[A-Za-z0-9.]+(\.onion)?$") + url = url.strip().removesuffix('/') + if url.startswith('http://'): + #print('URL starts with http') + # Removes the http:// + domain = url.split('/')[2] + if pattern.fullmatch(domain) is not None: + if len(domain.split('.')) > 3: + n_subdomians = len(domain.split('.')) + # Checks if there is more than 1 subdomain. "subdomain.url.onion" only + #print(f"This domain have more than one subdomain. There are {n_subdomians} subdomains") + return False + else: + if len(domain) < 62: + #print("Domain length is less than 62.") + return False + return True + elif pattern.fullmatch(domain) is None: + #print("Domain contains invalid character.") + #print(domain) + return False + else: + #print("Domain not valid") + return False + else: + #TODO : edit the url to make sure it has http:// at the beginning, in case if it's missing? (problem is that it only returns true or false) + #print("URL doesn't start http") + if pattern.fullmatch(url) is not None: + if len(url.split('.')) > 3: + n_subdomians = len(url.split('.')) + # Checks if there is more than 1 subdomain. "subdomain.url.onion" only + #print(f"This domain have more than one subdomain. There are {n_subdomians - 1} subdomains") + return False + else: + if len(url) < 62: + #print("Domain length is less than 62.") + return False + return True + elif pattern.fullmatch(url) is None: + #print("Domain contains invalid character.") + #print(url) + return False + else: + #print("Domain not valid") + return False + except Exception as e: + print(f"Error: {e}") + + + +if __name__ == '__main__': + main() diff --git a/www/participants/uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/unverified.csv b/www/participants/uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/unverified.csv index 341e47f..bce717f 100644 --- a/www/participants/uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/unverified.csv +++ b/www/participants/uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/unverified.csv @@ -1,6 +1,7 @@ Instance,Category,Name,URL,Sensitive,Description,Status,Score -webring.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion,Hackliberty,Hackliberty Gitea,http://vkp7367tcjpqdwwckigrdrvmwvispvbpg5rlsr2chjxvppfg7hipagyd.onion,,,, -webring.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion,Communities,Hackliberty main website,http://kj3wvs3wyfhm3uhhuqxlrhhcp6dneuau4mmvptlor27ghmrqx63fqnid.onion/,,,, -webring.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion,Forums,Hackliberty Forum,http://yw7nc56v4nsudvwewhmhhwltxpncedfuc43qbubj4nmwhdhwtiu4o6yd.onion/,,,, -uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,Infos and Links,Tor Taxi,http://tortaxi2dev6xjwbaydqzldruga77rrnth7yn2oqzjfmiuwn5h6vsk2a4syd.onion/,✔️,List of links to go to popular darknet places,✔️,0.0 +uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,MoneroNodes,MoneroNode1,http://24outdkavkxmwldmnn4lgzfh4uz2yvbcv3mubbpkubsuab435yaqwxqd.onion:18081,,,✔️,100.0 +webring.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion,Forums,Hackliberty Forum,http://yw7nc56v4nsudvwewhmhhwltxpncedfuc43qbubj4nmwhdhwtiu4o6yd.onion/,,,✔️,100.0 uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,Infos and Links,Tor Taxi,http://tortaxi2dev6xjwbaydqzla77rrnth7yn2oqzjfmiuwn5h6vsk2a4syd.onion/,,List of links to go to popular darknet places,✔️,100.0 +webring.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion,Hackliberty,Hackliberty Gitea,http://vkp7367tcjpqdwwckigrdrvmwvispvbpg5rlsr2chjxvppfg7hipagyd.onion,,,✔️,100.0 +webring.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion,Communities,Hackliberty main website,http://kj3wvs3wyfhm3uhhuqxlrhhcp6dneuau4mmvptlor27ghmrqx63fqnid.onion/,,,✔️,100.0 +uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,Infos and Links,Tor Taxi,http://tortaxi2dev6xjwbaydqzldruga77rrnth7yn2oqzjfmiuwn5h6vsk2a4syd.onion/,✔️,List of links to go to popular darknet places,❌,0.0 diff --git a/www/participants/uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/webring-participants.csv b/www/participants/uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/webring-participants.csv index 1aa88c3..713c776 100644 --- a/www/participants/uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/webring-participants.csv +++ b/www/participants/uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/webring-participants.csv @@ -1,25 +1,25 @@ Name,URL,Description,Trusted,Status,Score -,webring.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion,,,, -,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,✔️,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,First instance,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,First webring participant,,, -Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,New webring participant,,, +,webring.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion,,,✔️,100.0 +,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,✔️,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,First instance,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,First webring participant,,✔️,100.0 +Nowhere,uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,New webring participant,,✔️,100.0 Nowhere,http://uptime.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion,Darknet Webring Onion Participant,✔️,✔️,100.0