From c7074e4428114608fb0d6a5bce521d6375f9735b Mon Sep 17 00:00:00 2001
From: valuer <valuer@noreply.localhost>
Date: Thu, 8 May 2025 00:17:10 +0200
Subject: [PATCH] lantern.py option 1: when adding a new website, delete
 existing same urls ones

Fix issue #43
---
 scripts/lantern.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/scripts/lantern.py b/scripts/lantern.py
index 43e5c8c..a420f6a 100644
--- a/scripts/lantern.py
+++ b/scripts/lantern.py
@@ -229,8 +229,28 @@ Maintenance:
                     newrow=[instance,category,name,url,sensi,desc,'YES','100']
                     print_colors(f"[+] NEWROW= {newrow}")
                     # (rest is automatic: status, score, instance is = '' because it is your own instance)
-                    # TODO check if the entry doesnt already exist in verified.csv and in unverified.csv
-                    # if it doesnt exist, add it into unverified.csv
+                    # delete existing entries in verified.csv
+                    vdf_same_url_filter = vdf["URL"] == url # check for same url
+                    vdf_same_url_filter_count = vdf_same_url_filter.sum() # total url matches
+                    if vdf_same_url_filter_count > 0:
+                        print(f"Found {vdf_same_url_filter_count} row(s) with the same url in verified.csv")
+                        for index, row in vdf[vdf_same_url_filter].iterrows():
+                            print_colors(f"[+] ROW[{index}]= {list(row)}")
+                        vdf = vdf[~vdf_same_url_filter].reset_index(drop=True) # keep only entries that do not match filter
+                        print(f"Deleted {vdf_same_url_filter_count} row(s) with the same url in verified.csv")
+                        if desc == '': # if the description is empty = it means that it goes in unverified.csv, so save modified verified.csv file now
+                            vdf.to_csv(verifiedcsvfile, index=False)
+                    # delete existing entries in unverified.csv
+                    uvdf_same_url_filter = uvdf["URL"] == url # check for same url
+                    uvdf_same_url_filter_count = uvdf_same_url_filter.sum() # total url matches
+                    if uvdf_same_url_filter_count > 0:
+                        print(f"Found {uvdf_same_url_filter_count} row(s) with the same url in unverified.csv")
+                        for index, row in uvdf[uvdf_same_url_filter].iterrows():
+                            print_colors(f"[+] ROW[{index}]= {list(row)}")
+                        uvdf = uvdf[~uvdf_same_url_filter].reset_index(drop=True) # keep only entries that do not match filter
+                        print(f"Deleted {uvdf_same_url_filter_count} row(s) with the same url in unverified.csv")
+                        if desc != '': # if the description isnt empty = it means that it goes in verified.csv, so save modified unverified.csv file now
+                            uvdf.to_csv(unverifiedcsvfile, index=False)
                     if desc == '': # if the description is empty = it means that it goes in unverified.csv 
                         print("Adding new row in unverified.csv since description is empty")
                         uvdf.loc[-1] = newrow  # adding a row