oxeo0/darknet-lantern
1
0
Fork 0
mirror of http://git.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/nihilist/darknet-lantern.git synced 2025-05-16 12:16:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix option 9 duplicates

Browse source
This commit is contained in:
root 2025-05-04 20:24:36 +02:00
parent 997617abe6
commit 7c3c820a3b
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
scripts/lantern.py
View file

@ -1122,7 +1122,9 @@ Maintenance:
try:
csvdf = pd.read_csv(csvfilepath)
print_colors(f"Removing duplicates in {csvfilepath}")
csvdf = csvdf.drop_duplicates(subset=['URL'])
#print_colors(f"{csvdf[['URL']]}")
csvdf = csvdf.drop_duplicates(subset=['URL'], keep="first", inplace=False)
#print_colors(f"{csvdf[['URL']]}")
csvdf.to_csv(csvfilepath, index=False)
print_colors(f"Cleaned data:\n{csvdf[['URL']]}")
except FileNotFoundError: