mirror of
http://git.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/nihilist/darknet-lantern.git
synced 2025-07-01 18:56:40 +00:00
FINished adding the trusted and the refactoring of 6
This commit is contained in:
doctor_dev
parent
8b0ba4833f
commit
14231fff92
5 changed files with 631 additions and 321 deletions
401
scripts/utils.py
401
scripts/utils.py
|
|
@ -19,7 +19,7 @@ RESET = '\033[m'
|
|||
|
||||
def get_current_instance():
|
||||
"""
|
||||
Checks if all URL files are actually reachable via Tor
|
||||
Get the current host instance
|
||||
|
||||
Returns:
|
||||
str: the local instance onion url
|
||||
|
|
@ -137,11 +137,15 @@ def is_participant_reachable(instance):
|
|||
"""
|
||||
Checks if all URL files are actually reachable via Tor
|
||||
|
||||
Parameters:
|
||||
instance (str): The participant onion address
|
||||
Parameters
|
||||
----------
|
||||
instance : str
|
||||
The participant onion address
|
||||
|
||||
Returns:
|
||||
Boolean: False if any file is unreachable, True if all are reachable
|
||||
Returns
|
||||
-------
|
||||
Bool
|
||||
False if any file is unreachable, True if all are reachable
|
||||
"""
|
||||
|
||||
url = generate_participant_url(instance)
|
||||
|
|
@ -268,13 +272,17 @@ def send_server_checks(url: str) -> tuple[str, str, str]:
|
|||
|
||||
def is_row_valid(row):
|
||||
"""
|
||||
validates dataframe row to check if all field are valid
|
||||
Validates dataframe row to check if all field are valid
|
||||
|
||||
Parameters:
|
||||
row (dict): dataframe row
|
||||
Parameters
|
||||
----------
|
||||
row : dict
|
||||
Dataframe row
|
||||
|
||||
Returns:
|
||||
Boolean: True if row is valid, False if row isn't valid
|
||||
Returns
|
||||
-------
|
||||
Bool
|
||||
True if row is valid, False if row isn't valid
|
||||
"""
|
||||
try:
|
||||
return (
|
||||
|
|
@ -295,18 +303,23 @@ def is_row_valid(row):
|
|||
|
||||
def merge_verification_df(receiving_df, merging_df):
|
||||
"""
|
||||
merges 2 dataframes of type verified or unverified (do not merge duplications by name or url)
|
||||
Merges 2 dataframes of type verified or unverified (do not merge duplications by name or url)
|
||||
|
||||
Parameters:
|
||||
receiving_df (Dataframe): dataframe we want to receive the data
|
||||
merging_df (Dataframe): dataframe we want to merge into the receiving dataframe
|
||||
Parameters
|
||||
----------
|
||||
receiving_df : pd.DataFrame
|
||||
Dataframe we want to receive the data
|
||||
merging_df : pd.DataFrame
|
||||
Dataframe we want to merge into the receiving dataframe
|
||||
|
||||
Returns:
|
||||
Dataframe: the combined dataframe will be returned
|
||||
--------
|
||||
pd.DataFrame
|
||||
The combined dataframe will be returned
|
||||
"""
|
||||
try:
|
||||
filtered_df = merging_df[~((merging_df['URL'].isin(receiving_df['URL'])) | merging_df['Name'].isin(receiving_df['Name']))]
|
||||
|
||||
|
||||
if filtered_df.empty:
|
||||
return receiving_df
|
||||
|
||||
|
|
@ -321,13 +334,17 @@ def merge_verification_df(receiving_df, merging_df):
|
|||
|
||||
def remove_duplications(df):
|
||||
"""
|
||||
remove url and name duplications from the dataframe
|
||||
Remove url and name duplications from the dataframe
|
||||
|
||||
Parameters:
|
||||
df (Dataframe): the dataframe to remove duplications from
|
||||
Parameters
|
||||
----------
|
||||
df : pd.DataFrame
|
||||
The dataframe to remove duplications from
|
||||
|
||||
Returns:
|
||||
Dataframe: the dataframe after all duplications were removed
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
The dataframe after all duplications were removed
|
||||
"""
|
||||
try:
|
||||
df = df.drop_duplicates(subset='Name')
|
||||
|
|
@ -340,15 +357,21 @@ def remove_duplications(df):
|
|||
|
||||
def remove_cross_dataframe_replications(main_df, sub_df):
|
||||
"""
|
||||
remove replications from sub_df that exist in main_df
|
||||
Remove replications from sub_df that exist in main_df
|
||||
|
||||
Parameters:
|
||||
main_df (Dataframe): the dataframe to keep replications
|
||||
sub_df (Dataframe): the dataframe to remove replications
|
||||
Parameters
|
||||
----------
|
||||
main_df : pd.DataFrame
|
||||
The dataframe to keep replications
|
||||
sub_df : DataFrame
|
||||
The dataframe to remove replications
|
||||
|
||||
Returns:
|
||||
Dataframe: the main_df with removed duplications
|
||||
Dataframe: the sub_df with removed duplications and removed replications
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
The main_df with removed duplications
|
||||
pd.DataFrame
|
||||
The sub_df with removed duplications and removed replications
|
||||
"""
|
||||
try:
|
||||
|
||||
|
|
@ -365,24 +388,150 @@ def remove_cross_dataframe_replications(main_df, sub_df):
|
|||
|
||||
return main_df, sub_df
|
||||
|
||||
def add_word_to_blacklist(word):
|
||||
"""
|
||||
Add a new word to the blacklist
|
||||
|
||||
Parameters
|
||||
----------
|
||||
word : str
|
||||
The new word we want to add to the blacklist
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
True if word is in the blacklist or added, False if fails
|
||||
"""
|
||||
|
||||
try:
|
||||
local_blacklist_df = get_local_blacklist()
|
||||
|
||||
if word not in local_blacklist_df['blacklisted-words'].values:
|
||||
|
||||
local_blacklist_df.loc[len(local_blacklist_df)] = [word]
|
||||
|
||||
save_local_blacklist(local_blacklist_df)
|
||||
|
||||
else:
|
||||
print_colors('[+] Word already exists in the blacklist')
|
||||
|
||||
except Exception as err:
|
||||
print_colors('[-] Adding word to the blacklist failed',is_error=True)
|
||||
|
||||
return local_blacklist_df
|
||||
|
||||
def remove_word_from_blacklist(word):
|
||||
"""
|
||||
Remove a word from the blacklist
|
||||
|
||||
Parameters
|
||||
----------
|
||||
word : str
|
||||
The word we want to remove from the blacklist
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
True if word is not in the blacklist or removed, False if fails
|
||||
"""
|
||||
|
||||
try:
|
||||
local_blacklist_df = get_local_blacklist()
|
||||
|
||||
if word in local_blacklist_df['blacklisted-words'].values:
|
||||
|
||||
local_blacklist_df = local_blacklist_df[local_blacklist_df['blacklisted-words'] != word]
|
||||
|
||||
save_local_blacklist(local_blacklist_df)
|
||||
|
||||
else:
|
||||
print_colors('[+] Word wasn\'t found on the blacklist')
|
||||
|
||||
except Exception as err:
|
||||
print_colors('[-] Removing word from the blacklist failed',is_error=True)
|
||||
|
||||
return local_blacklist_df
|
||||
|
||||
|
||||
def transfer_rows_by_instance(target_df, source_df, participant_instance):
|
||||
"""
|
||||
Transfer rows from one dataframe to another by instance condition
|
||||
|
||||
Parameters
|
||||
----------
|
||||
target_df pd.DataFrame
|
||||
The dataframe i want to copy into
|
||||
source_df pd.DataFrame
|
||||
The dataframe i want to cut out of
|
||||
participant_instance : str
|
||||
The participant's instance onion address
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
The target_df with the new rows
|
||||
pd.DataFrame
|
||||
The source_df with the removed rows
|
||||
"""
|
||||
|
||||
try:
|
||||
mask = source_df['Instance'] == participant_instance
|
||||
|
||||
target_df = pd.concat([target_df, source_df[mask]])
|
||||
|
||||
source_df = source_df[~mask]
|
||||
|
||||
except Exception as err:
|
||||
print_colors('[-] Transferring rows by instance failed',is_error=True)
|
||||
|
||||
return target_df, source_df
|
||||
|
||||
def save_local_blacklist(blacklist_df):
|
||||
"""
|
||||
Saves the local blacklist
|
||||
|
||||
Parameters
|
||||
----------
|
||||
blacklist_df : pd.DataFrame
|
||||
Dataframe of the blacklist
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
True if successful, False if not
|
||||
"""
|
||||
|
||||
try:
|
||||
save_dataframe(blacklist_df, f'{conf.LOCAL_DIR}blacklist.csv')
|
||||
|
||||
return True
|
||||
|
||||
except Exception as err:
|
||||
print_colors('[-] Saving blacklist failed',is_error=True)
|
||||
return False
|
||||
|
||||
###TODO: can later remove the inputs and have a "global" local verified and unverified or a class of the local(lantern host) participant
|
||||
def save_local_verified_and_unverified(verified_df, unverified_df):
|
||||
"""
|
||||
saves the local verified and unverified
|
||||
Saves the local verified and unverified
|
||||
|
||||
Parameters:
|
||||
verified_df (Dataframe): local verified rows dataframe
|
||||
unverified_df (Dataframe): local unverified rows dataframe
|
||||
Parameters
|
||||
----------
|
||||
verified_df : pd.DataFrame
|
||||
Local verified rows dataframe
|
||||
unverified_df : DataFrame
|
||||
Local unverified rows dataframe
|
||||
|
||||
Returns:
|
||||
bool: True if successful, False if not
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
True if successful, False if not
|
||||
"""
|
||||
try:
|
||||
current_instance = get_current_instance() + '/'
|
||||
|
||||
verified_df.to_csv(f'{conf.PARTICIPANT_DIR}{current_instance}verified.csv', index=False)
|
||||
save_dataframe(verified_df, f'{conf.LOCAL_DIR}verified.csv')
|
||||
|
||||
unverified_df.to_csv(f'{conf.PARTICIPANT_DIR}{current_instance}unverified.csv', index=False)
|
||||
save_dataframe(unverified_df, f'{conf.LOCAL_DIR}unverified.csv')
|
||||
|
||||
print_colors('[+] Verified and unverified saved successfully')
|
||||
|
||||
|
|
@ -394,21 +543,28 @@ def save_local_verified_and_unverified(verified_df, unverified_df):
|
|||
|
||||
def save_local_participant_verified_and_unverified(verified_df, unverified_df, participant):
|
||||
"""
|
||||
saves the local verified and unverified of a participant
|
||||
Saves the local verified and unverified of a participant
|
||||
|
||||
Parameters:
|
||||
verified_df (Dataframe): local verified rows dataframe
|
||||
unverified_df (Dataframe): local unverified rows dataframe
|
||||
participant (str): participant's onion local path
|
||||
Parameters
|
||||
----------
|
||||
verified_df pd.DataFrame
|
||||
Local verified rows dataframe
|
||||
unverified_df pd.DataFrame
|
||||
Local unverified rows dataframe
|
||||
participant : str
|
||||
Participant's onion local path
|
||||
|
||||
Returns:
|
||||
bool: True if successful, False if not
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
True if successful, False if not
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
verified_df.to_csv(f'{participant}verified.csv', index=False)
|
||||
save_dataframe(verified_df, f'{participant}verified.csv')
|
||||
|
||||
unverified_df.to_csv(f'{participant}unverified.csv', index=False)
|
||||
save_dataframe(unverified_df, f'{participant}unverified.csv')
|
||||
|
||||
print_colors('[+] Verified and unverified saved successfully')
|
||||
|
||||
|
|
@ -418,43 +574,82 @@ def save_local_participant_verified_and_unverified(verified_df, unverified_df, p
|
|||
print_colors('[-] Saving verified and unverified failed',is_error=True)
|
||||
return False
|
||||
|
||||
def save_dataframe(df, path):
|
||||
"""
|
||||
Saves a dataframe
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df : pd.DataFrame
|
||||
Dataframe wants to be saved
|
||||
|
||||
path : str
|
||||
Local path for the dataframe
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
True if saved, False if not
|
||||
"""
|
||||
|
||||
try:
|
||||
df.to_csv(path, index=False)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as err:
|
||||
return False
|
||||
|
||||
###################### Getters/Generators ######################
|
||||
def generate_participant_url(participant):
|
||||
"""
|
||||
generates url of the webring participant
|
||||
Generates url of the webring participant
|
||||
|
||||
Parameters:
|
||||
participant(str): participant's onion address/instance
|
||||
Parameters
|
||||
----------
|
||||
participant : str
|
||||
Participant's onion address/instance
|
||||
|
||||
Returns:
|
||||
str: the url of the webring participant
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The url of the webring participant
|
||||
"""
|
||||
|
||||
return f'http://{participant}/participants/{participant}/'
|
||||
|
||||
def generate_local_participant_dir(participant):
|
||||
"""
|
||||
generates local files path of the webring participant
|
||||
Generates local files path of the webring participant
|
||||
|
||||
Parameters:
|
||||
participant(str): participant's onion address/instance
|
||||
Parameters
|
||||
----------
|
||||
participant : str
|
||||
Participant's onion address/instance
|
||||
|
||||
Returns:
|
||||
str: the local path of the webring participant's files
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The local path of the webring participant's files
|
||||
"""
|
||||
|
||||
return f'{conf.PARTICIPANT_DIR}{participant}/'
|
||||
|
||||
def get_participant_local_verified_and_unverified(participant):
|
||||
"""
|
||||
reads the local verified csv and the local unverified csv of a participant
|
||||
Reads the local verified csv and the local unverified csv of a participant
|
||||
|
||||
Parameters:
|
||||
participant (str): participant's local files path
|
||||
Parameters
|
||||
----------
|
||||
participant : str
|
||||
Participant's local files path
|
||||
|
||||
Returns:
|
||||
verified_df(Dataframe): verified.csv as dataframe
|
||||
unverified_df(Dataframe): unverified.csv as dataframe
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
verified.csv as dataframe
|
||||
pd.DataFrame
|
||||
unverified.csv as dataframe
|
||||
"""
|
||||
|
||||
try:
|
||||
|
|
@ -462,24 +657,26 @@ def get_participant_local_verified_and_unverified(participant):
|
|||
|
||||
except FileNotFoundError:
|
||||
print_colors("[-] File not found: verified.csv", is_error=True)
|
||||
return pd.Dataframe(), pd.Dataframe()
|
||||
return pd.DataFrame(), pd.DataFrame()
|
||||
|
||||
try:
|
||||
unverified_df = pd.read_csv(f'{participant}unverified.csv')
|
||||
|
||||
except FileNotFoundError:
|
||||
print_colors("[-] Participant File not found: unverified.csv", is_error=True)
|
||||
return pd.Dataframe(), pd.Dataframe()
|
||||
return pd.DataFrame(), pd.DataFrame()
|
||||
|
||||
return verified_df, unverified_df
|
||||
|
||||
|
||||
def get_official_participants():
|
||||
"""
|
||||
reads all the official webring participants
|
||||
Reads all the official webring participants
|
||||
|
||||
Returns:
|
||||
list: list of all the official webring participants
|
||||
Returns
|
||||
-------
|
||||
list
|
||||
List of all the official webring participants
|
||||
"""
|
||||
|
||||
try:
|
||||
|
|
@ -491,58 +688,78 @@ def get_official_participants():
|
|||
except Exception as err:
|
||||
print_colors('[-] Couldn\'t read official webring participants file',is_error=True )
|
||||
|
||||
def get_local_blacklist_and_sensitive():
|
||||
def get_local_blacklist():
|
||||
"""
|
||||
reads the local blacklisted words and the local sensitive words
|
||||
Reads the local blacklist
|
||||
|
||||
Returns:
|
||||
blacklist(list): list of all the words that are blacklisted
|
||||
sensitive_list(list): list of all the words that are sensitive
|
||||
Returns
|
||||
-------
|
||||
blacklist_df : pd.DataFrame
|
||||
Dataframe of the blacklist
|
||||
"""
|
||||
try:
|
||||
current_instance = get_current_instance() + '/'
|
||||
try:
|
||||
blacklist_df = pd.read_csv(f'{conf.PARTICIPANT_DIR}{current_instance}blacklist.csv')
|
||||
blacklist = blacklist_df.iloc[:, 0].tolist()
|
||||
blacklist_df = pd.read_csv(f'{conf.LOCAL_DIR}blacklist.csv')
|
||||
|
||||
except FileNotFoundError:
|
||||
print_colors("[-] File not found: blacklist.csv", is_error=True)
|
||||
|
||||
|
||||
return blacklist_df
|
||||
|
||||
except Exception as err:
|
||||
print_colors('[-] Failed reading the blacklist words file',is_error=True)
|
||||
|
||||
return pd.DataFrame()
|
||||
|
||||
def get_local_sensitive():
|
||||
"""
|
||||
Reads the local sensitive words
|
||||
|
||||
Returns
|
||||
-------
|
||||
sensitive_list list
|
||||
List of all the words that are sensitive
|
||||
"""
|
||||
try:
|
||||
|
||||
try:
|
||||
sensitive_df = pd.read_csv(f'{conf.PARTICIPANT_DIR}{current_instance}sensitive.csv')
|
||||
sensitive_list = sensitive_df.iloc[:, 0].tolist()
|
||||
sensitive_df = pd.read_csv(f'{conf.LOCAL_DIR}sensitive.csv')
|
||||
|
||||
except FileNotFoundError:
|
||||
print_colors("[-] File not found: sensitive.csv", is_error=True)
|
||||
|
||||
|
||||
return blacklist, sensitive_list
|
||||
return sensitive_df
|
||||
|
||||
|
||||
except Exception as err:
|
||||
print_colors('[-] Failed reading the blacklist and sensitive words file',is_error=True)
|
||||
print_colors('[-] Failed reading the sensitive words file',is_error=True)
|
||||
|
||||
return [], []
|
||||
return pd.DataFrame()
|
||||
|
||||
def get_local_verified_and_unverified():
|
||||
"""
|
||||
reads the local verified csv and the local unverified csv of the instance
|
||||
Reads the local verified csv and the local unverified csv of the instance
|
||||
|
||||
Returns:
|
||||
verified_df(Dataframe): verified.csv as dataframe
|
||||
unverified_df(Dataframe): unverified.csv as dataframe
|
||||
Returns
|
||||
-------
|
||||
verified_df : pd.DataFrame
|
||||
verified.csv as dataframe
|
||||
unverified_df : pd.DataFrame
|
||||
unverified.csv as dataframe
|
||||
"""
|
||||
|
||||
try:
|
||||
current_instance = get_current_instance() + '/'
|
||||
|
||||
try:
|
||||
verified_df = pd.read_csv(f'{conf.PARTICIPANT_DIR}{current_instance}verified.csv')
|
||||
verified_df = pd.read_csv(f'{conf.LOCAL_DIR}verified.csv')
|
||||
|
||||
except FileNotFoundError:
|
||||
print_colors("[-] File not found: verified.csv", is_error=True)
|
||||
|
||||
try:
|
||||
unverified_df = pd.read_csv(f'{conf.PARTICIPANT_DIR}{current_instance}unverified.csv')
|
||||
unverified_df = pd.read_csv(f'{conf.LOCAL_DIR}unverified.csv')
|
||||
|
||||
except FileNotFoundError:
|
||||
print_colors("[-] File not found: unverified.csv", is_error=True)
|
||||
|
|
@ -556,10 +773,12 @@ def get_local_verified_and_unverified():
|
|||
|
||||
def get_local_webring_participants():
|
||||
"""
|
||||
make sure the official participants are registered in the webring csv file
|
||||
Make sure the official participants are registered in the webring csv file
|
||||
|
||||
Returns:
|
||||
Dataframe: the verified local webring participants dataframe
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
The verified local webring participants dataframe
|
||||
"""
|
||||
|
||||
try:
|
||||
|
|
@ -572,7 +791,7 @@ def get_local_webring_participants():
|
|||
new_row = [{'Name': '','URL': participant,'Description': '','Trusted': 'NO','Status': '','Score': ''}]
|
||||
webring_df = pd.concat([webring_df, pd.DataFrame(new_row)], ignore_index=True)
|
||||
|
||||
webring_df.to_csv(conf.LOCAL_DIR + conf.WEBRING_CSV_FILE, index=False)
|
||||
save_dataframe(webring_df, conf.LOCAL_DIR + conf.WEBRING_CSV_FILE)
|
||||
|
||||
return webring_df
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue