oxeo0/darknet-lantern
1
0
Fork 0
mirror of http://git.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/nihilist/darknet-lantern.git synced 2025-05-16 04:06:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix lantern

Browse source
This commit is contained in:
root 2025-05-09 14:42:57 +02:00
parent f06ae54ed5
commit 5204164f94
1 changed files with 8 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

8
scripts/lantern.py
View file

@ -674,6 +674,7 @@ Maintenance:
uvdf = pd.read_csv(unverifiedcsvfile, on_bad_lines='skip') uvdf = pd.read_csv(unverifiedcsvfile, on_bad_lines='skip')
# TODO DELETE ALL DUPLICATES OF UVDF ! # TODO DELETE ALL DUPLICATES OF UVDF !
uvdf = uvdf.drop_duplicates(subset=['URL'], keep="first", inplace=False) uvdf = uvdf.drop_duplicates(subset=['URL'], keep="first", inplace=False)
uvdf.to_csv(unverifiedcsvfile, index=False)
filter_uvdf= uvdf[uvdf.URL.str.contains(filterterm,na=False)] filter_uvdf= uvdf[uvdf.URL.str.contains(filterterm,na=False)]
if len(filter_uvdf.index) == 0 and len(filter_vdf.index) == 0: if len(filter_uvdf.index) == 0 and len(filter_vdf.index) == 0:
newrow=row newrow=row
@ -709,6 +710,13 @@ Maintenance:
csvdf.drop(i, inplace= True) csvdf.drop(i, inplace= True)
csvdf.to_csv(csvfilepath, index=False) csvdf.to_csv(csvfilepath, index=False)
rows2delete= [] # it is an empty list at first rows2delete= [] # it is an empty list at first
# TODO DELETE ALL DUPLICATES OF UVDF !
uvdf = uvdf.sort_index() # sorting by index
uvdf = uvdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
uvdf = uvdf.drop_duplicates(subset=['URL'], keep="first", inplace=False)
uvdf.to_csv(unverifiedcsvfile, index=False)
break break