working_dedupe.py

working_dedupe.py · 2.4 KiB · Python Raw

#!/usr/bin/env python3
 
# dedup.py
# Removes duplicates from Bitwarden export .csv
# 2019-02-09 5erif
 
import sys
import hashlib
from urllib.parse import urlparse
 
# Field ordinals in Bitwarden CSV
FOLDER   = 0
FAVORITE = 1
TYPE     = 2
NAME     = 3
NOTES    = 4
FIELDS   = 5
REPROMPT = 6
URI      = 7
USERNAME = 8
PASSWORD = 9
TOTP     = 10
 
DEBUG = False
def main(argv):
 
    if len(argv) < 1:
        print('Missing input file path')
        sys.exit(1)
        
    in_file_path  = argv[0]
    out_file_path = in_file_path[0:(len(in_file_path)-4)]+'_out.csv'
    rem_file_path = in_file_path[0:(len(in_file_path)-4)]+'_rem.csv'
    completed_lines_hash = set()
    line_number   = -1
    write_count   = 0
    cache         = ''
    
    out_file = open(out_file_path, 'w', encoding = 'utf8')
    rem_file = open(rem_file_path, 'w', encoding = 'utf8')
    for line in open(in_file_path, 'r', encoding = 'utf8'):
        line_number += 1
        fields = line.split(',')
        if len(fields) < 10:
            # Add previous line if short
            line = cache.strip('\n') + line
            cache = line
            fields = line.split(',')
            if len(fields) > 9:
                print(f'Recovered with line {line_number}:\n{line}')
                cache = ''
            else:
                print(f'Missing fields in line {line_number}:\n{line}')
                rem_file.write(line)
                continue
        else:
            cache = ''
        if line_number != 0:
            domain = urlparse(fields[URI]).netloc
            if DEBUG:
                print(f'URI: {fields[URI]}')
            if len(domain) > 0:
                fields[URI] = domain
        token = fields[URI] + fields[USERNAME] + fields[PASSWORD]
        hashValue = hashlib.md5(token.rstrip().encode('utf-8')).hexdigest()
        if hashValue not in completed_lines_hash:
            out_file.write(line)
            completed_lines_hash.add(hashValue)
            write_count += 1
        else: 
            rem_file.write(line)
            # Uncomment for verbose mode
            #print(f'Skipping duplicate on line {line_number}:\n{line}')
    out_file.close()
    rem_file.close()
    
    dup_count = line_number - write_count
    print(f'\nOutput file: {out_file_path}\n{write_count} unique entries saved')
    print(f'\n{dup_count} duplicates saved to {rem_file_path}')
 
if __name__ == "__main__":
   main(sys.argv[1:])

1	#!/usr/bin/env python3
2
3	# dedup.py
4	# Removes duplicates from Bitwarden export .csv
5	# 2019-02-09 5erif
6
7	import sys
8	import hashlib
9	from urllib.parse import urlparse
10
11	# Field ordinals in Bitwarden CSV
12	FOLDER = 0
13	FAVORITE = 1
14	TYPE = 2
15	NAME = 3
16	NOTES = 4
17	FIELDS = 5
18	REPROMPT = 6
19	URI = 7
20	USERNAME = 8
21	PASSWORD = 9
22	TOTP = 10
23
24	DEBUG = False
25	def main(argv):
26
27	if len(argv) < 1:
28	print('Missing input file path')
29	sys.exit(1)
30
31	in_file_path = argv[0]
32	out_file_path = in_file_path[0:(len(in_file_path)-4)]+'_out.csv'
33	rem_file_path = in_file_path[0:(len(in_file_path)-4)]+'_rem.csv'
34	completed_lines_hash = set()
35	line_number = -1
36	write_count = 0
37	cache = ''
38
39	out_file = open(out_file_path, 'w', encoding = 'utf8')
40	rem_file = open(rem_file_path, 'w', encoding = 'utf8')
41	for line in open(in_file_path, 'r', encoding = 'utf8'):
42	line_number += 1
43	fields = line.split(',')
44	if len(fields) < 10:
45	# Add previous line if short
46	line = cache.strip('\n') + line
47	cache = line
48	fields = line.split(',')
49	if len(fields) > 9:
50	print(f'Recovered with line {line_number}:\n{line}')
51	cache = ''
52	else:
53	print(f'Missing fields in line {line_number}:\n{line}')
54	rem_file.write(line)
55	continue
56	else:
57	cache = ''
58	if line_number != 0:
59	domain = urlparse(fields[URI]).netloc
60	if DEBUG:
61	print(f'URI: {fields[URI]}')
62	if len(domain) > 0:
63	fields[URI] = domain
64	token = fields[URI] + fields[USERNAME] + fields[PASSWORD]
65	hashValue = hashlib.md5(token.rstrip().encode('utf-8')).hexdigest()
66	if hashValue not in completed_lines_hash:
67	out_file.write(line)
68	completed_lines_hash.add(hashValue)
69	write_count += 1
70	else:
71	rem_file.write(line)
72	# Uncomment for verbose mode
73	#print(f'Skipping duplicate on line {line_number}:\n{line}')
74	out_file.close()
75	rem_file.close()
76
77	dup_count = line_number - write_count
78	print(f'\nOutput file: {out_file_path}\n{write_count} unique entries saved')
79	print(f'\n{dup_count} duplicates saved to {rem_file_path}')
80
81	if __name__ == "__main__":
82	main(sys.argv[1:])