Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Write config to & read config from JSON file (+ Chromium) #45

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Write config to & read config from JSON file (+ Chromium) #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
93d7a7b
Browser paths returns browser & profile name (to be written to a conf…
kchawla-pi Oct 7, 2019
328adea
Merge branch 'master' of https://github.com/kchawla-pi/united-states-…
kchawla-pi Oct 20, 2019
adfdb15
Browser info is now written to and read from a JSON file
kchawla-pi Oct 29, 2019
e1c1c4d
Corrected tests to work with function changes
kchawla-pi Oct 29, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions tests/tests/test_browser_exceptions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ def test_firefox_two_proflies_one_invalid(tests_root):
profile_names = ['test_profile1_JUNK', 'test_profile2']
file_name = 'places.sqlite'
expected_browser_paths = {
'test_profile2': Path(profile_rootpath, 'z786c76dv78.test_profile2')
('firefox','test_profile2'): str(Path(profile_rootpath,
'z786c76dv78.test_profile2')
)
}
actual_browser_paths = make_browser_paths(browser=browser_name,
profile_root=profile_rootpath,
Expand Down Expand Up @@ -161,7 +163,9 @@ def test_chrome_two_proflies_one_invalid(tests_root):
profile_names = ['Profile 1 JUNK', 'Profile 2']
file_name = 'History'
expected_browser_paths = {
'Profile 2': Path(profile_rootpath, 'Profile 2')
('chrome', 'Profile 2'): str(Path(profile_rootpath,
'Profile 2')
)
}
actual_browser_paths = make_browser_paths(browser=browser_name,
profile_root=profile_rootpath,
Expand Down
67 changes: 43 additions & 24 deletions tests/tests/test_browser_paths.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ def test_browser_firefox_make_paths_during_init_one_profile(tests_root):
profiles=[profile_name],
)
expected_paths = {
'test_profile1': Path(profile_rootpath, 't87e6f86.test_profile1')
('firefox', 'test_profile1'): str(Path(profile_rootpath, 't87e6f86.test_profile1'))
}

assert profile1_browser_paths.keys() == expected_paths.keys()

for profile_name in profile1_browser_paths:
assert profile1_browser_paths[profile_name] == expected_paths[
profile_name]
for browser_profile in profile1_browser_paths:
assert profile1_browser_paths[browser_profile] == expected_paths[
browser_profile]


def test_browser_firefox_make_paths_during_init_two_profiles(tests_root):
Expand All @@ -33,14 +33,19 @@ def test_browser_firefox_make_paths_during_init_two_profiles(tests_root):
profiles=profile_names,
)
expected_paths = {
'test_profile1': Path(profile_rootpath, 't87e6f86.test_profile1'),
'test_profile2': Path(profile_rootpath, 'z786c76dv78.test_profile2'),
('firefox', 'test_profile1'): str(Path(profile_rootpath,
't87e6f86.test_profile1')
),
('firefox', 'test_profile2'): str(Path(profile_rootpath,
'z786c76dv78.test_profile2')
),
}

assert browser_profile1.keys() == expected_paths.keys()

for profile_name in browser_profile1:
assert browser_profile1[profile_name] == expected_paths[profile_name]
for browser_profile in browser_profile1:
assert browser_profile1[browser_profile] == expected_paths[
browser_profile]


def test_browser_firefox_make_paths_during_init_all_profiles(tests_root):
Expand All @@ -53,15 +58,19 @@ def test_browser_firefox_make_paths_during_init_all_profiles(tests_root):
profiles=profile_names,
)
expected_paths = {
'test_profile1': Path(profile_rootpath, 't87e6f86.test_profile1'),
'test_profile2': Path(profile_rootpath, 'z786c76dv78.test_profile2'),
('firefox', 'test_profile1'): str(Path(profile_rootpath,
't87e6f86.test_profile1')
),
('firefox', 'test_profile2'): str(Path(profile_rootpath,
'z786c76dv78.test_profile2')
),
}

assert browser_profile1.keys() == expected_paths.keys()

for profile_name in browser_profile1:
assert browser_profile1[profile_name] == expected_paths[
profile_name]
for browser_profile in browser_profile1:
assert browser_profile1[browser_profile] == expected_paths[
browser_profile]


def test_browser_chrome_make_paths_during_init_one_profile(tests_root):
Expand All @@ -73,13 +82,15 @@ def test_browser_chrome_make_paths_during_init_one_profile(tests_root):
profile_root=profile_rootpath,
profiles=[profile_name],
)
expected_paths = {'Profile 1': Path(profile_rootpath, 'Profile 1')}
expected_paths = {('chrome', 'Profile 1'): str(Path(profile_rootpath,
'Profile 1'))
}

assert browser_profile1.keys() == expected_paths.keys()

for profile_name in browser_profile1:
assert browser_profile1[profile_name] == expected_paths[
profile_name]
for browser_profile in browser_profile1:
assert browser_profile1[browser_profile] == expected_paths[
browser_profile]


def test_browser_chrome_make_paths_during_init_two_profiles(tests_root):
Expand All @@ -91,15 +102,19 @@ def test_browser_chrome_make_paths_during_init_two_profiles(tests_root):
profile_root=profile_rootpath,
profiles=profile_names,
)
expected_paths = {'Profile 1': Path(profile_rootpath, 'Profile 1'),
'Profile 2': Path(profile_rootpath, 'Profile 2'),
expected_paths = {('chrome', 'Profile 1'): str(Path(profile_rootpath,
'Profile 1')
),
('chrome', 'Profile 2'): str(Path(profile_rootpath,
'Profile 2')
),
}

assert browser_profile1.keys() == expected_paths.keys()

for profile_name in browser_profile1:
assert browser_profile1[profile_name] == expected_paths[
profile_name]
for browser_profile in browser_profile1:
assert browser_profile1[browser_profile] == expected_paths[
browser_profile]


def test_browser_chrome_make_paths_during_init_all_profiles(tests_root):
Expand All @@ -111,8 +126,12 @@ def test_browser_chrome_make_paths_during_init_all_profiles(tests_root):
profile_root=profile_rootpath,
profiles=profile_names,
)
expected_paths = {'Profile 1': Path(profile_rootpath, 'Profile 1'),
'Profile 2': Path(profile_rootpath, 'Profile 2'),
expected_paths = {('chrome', 'Profile 1'): str(Path(profile_rootpath,
'Profile 1')
),
('chrome', 'Profile 2'): str(Path(profile_rootpath,
'Profile 2')
),
}

assert browser_profile1.keys() == expected_paths.keys()
Expand Down
34 changes: 18 additions & 16 deletions tests/tests/test_db_merge.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,10 @@ def test_make_records_yielder(tests_root):

def test_rename_existing_db():
with tempfile.TemporaryDirectory() as tmp_dir:
app_path, output_db = make_paths(app_path=tmp_dir,
db_name='test_combi_db',
)
app_path, output_db, config_filepath = make_paths(
app_path=tmp_dir,
db_name='test_combi_db',
)
expected_renamed_db_path = Path(tmp_dir, '_previous_test_combi_db')
output_db.write_text('junk')
assert output_db.exists()
Expand All @@ -153,10 +154,10 @@ def test_rename_existing_db():
def test_rename_existing_db_delete_existing_backup():
# case when previous db file backup exists
with tempfile.TemporaryDirectory() as tmp_dir:
app_path, output_db = make_paths(app_path=[tmp_dir, ''],
# case when path == [dirnames]
db_name='test_combi_db',
)
app_path, output_db, config_file = make_paths(app_path=[tmp_dir, ''],
# case when path == [dirnames]
db_name='test_combi_db',
)
expected_renamed_db_path = Path(tmp_dir, '_previous_test_combi_db')
output_db.write_text('junk')
expected_renamed_db_path.write_text('1')
Expand Down Expand Up @@ -191,9 +192,9 @@ def _test_write_record_(tmp_dir):
so clean up on Windows does not glitch
due to PermissionError with open file handles.
"""
app_path, output_db = make_paths(app_path=tmp_dir,
db_name='test_combi_db',
)
app_path, output_db, config_file = make_paths(app_path=tmp_dir,
db_name='test_combi_db',
)
tablename = 'junk_table'
write_records(records_yielders=mock_records_generator,
output_db=output_db,
Expand All @@ -218,9 +219,10 @@ def _test_write_record_(tmp_dir):

def test_write_records_improper_table_name():
with tempfile.TemporaryDirectory() as tmp_dir:
app_path, output_db = make_paths(app_path=tmp_dir,
db_name='test_combi_db',
)
app_path, output_db, config_file = make_paths(
app_path=tmp_dir,
db_name='test_combi_db',
)
browser_yielder = [{},{}]
tablename = 'junk _table'
with pytest.raises(ValueError) as excep:
Expand All @@ -237,9 +239,9 @@ def test_write_records_improper_table_name():
def test_write_db_path_to_file():
test_db_name = 'test_combi_db'
with tempfile.TemporaryDirectory() as tmp_dir:
app_path, output_db = make_paths(app_path=tmp_dir,
db_name='test_combi_db',
)
app_path, output_db, config_file = make_paths(app_path=tmp_dir,
db_name='test_combi_db',
)
write_db_path_to_file(output_db=output_db, output_dir=tmp_dir)
expected_output_path = Path(tmp_dir, 'AppData', 'merged_db_path.txt')
assert expected_output_path.exists()
Expand Down
21 changes: 13 additions & 8 deletions united_states_of_browsers/db_merge/browser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def make_browser_records_yielder(browser: Text,
:return: Generator of records
"""
paths = make_browser_paths(browser, profile_root, profiles)
for profile_name, profile_path in paths.items():
for (browser_name, profile_name), profile_path in paths.items():
filepath = Path(profile_path, filename)
profile_name = Path(profile_path).name
table_obj = Table(table=tablename,
Expand All @@ -60,7 +60,7 @@ def make_browser_records_yielder(browser: Text,
yield record


def _make_firefox_profile_paths(profile_root: PathInfo, profiles: Iterable[Text]) -> Mapping[Text, PathInfo]:
def _make_firefox_profile_paths(browser: Text, profile_root: PathInfo, profiles: Iterable[Text]) -> Mapping[Text, PathInfo]:
"""
Makes a dict of profile names and paths for Chrome broser.

Expand All @@ -74,18 +74,18 @@ def _make_firefox_profile_paths(profile_root: PathInfo, profiles: Iterable[Text]

get_profile_name = lambda dir_entry: str(dir_entry).split(sep='.', maxsplit=1)[1]
if profiles:
profilepaths = {profile: entry for entry in profile_root.iterdir()
profilepaths = {(browser, profile): entry for entry in profile_root.iterdir()
for profile in profiles
if '.' in entry.name and entry.is_dir() and entry.name.endswith(profile)
}
else:
profilepaths = {get_profile_name(entry): entry
profilepaths = {(browser, get_profile_name(entry)): entry
for entry in profile_root.iterdir()
if '.' in entry.name and entry.is_dir()}
return profilepaths


def _make_chrome_profile_paths(profile_root: PathInfo, profiles: Iterable[Text]) -> Mapping[Text, PathInfo]:
def _make_chrome_profile_paths(browser: Text, profile_root: PathInfo, profiles: Iterable[Text]) -> Mapping[Text, PathInfo]:
"""
Makes a dict of profile names and paths for Chrome broser.

Expand All @@ -97,12 +97,12 @@ def _make_chrome_profile_paths(profile_root: PathInfo, profiles: Iterable[Text])
:return: Dict of profile names and corresponding path.
"""
if profiles:
profilepaths = {entry.name: entry
profilepaths = {(browser, entry.name): entry
for profile_name in profiles
for entry in profile_root.iterdir()
if entry.name.endswith(profile_name)}
else:
profilepaths = {entry.name: entry for entry in profile_root.iterdir()
profilepaths = {(browser, entry.name): entry for entry in profile_root.iterdir()
if entry.name.startswith('Profile') or entry.name == 'Default'}
return profilepaths

Expand All @@ -121,12 +121,17 @@ def make_browser_paths(browser: Text, profile_root: PathInfo, profiles: Iterable
"""
make_path_chooser = {'firefox': _make_firefox_profile_paths, 'chrome': _make_chrome_profile_paths,
'opera': _make_chrome_profile_paths, 'vivaldi': _make_chrome_profile_paths,
'chromium': _make_chrome_profile_paths,
}
try:
profilepaths = make_path_chooser[browser](profile_root, profiles)
profilepaths = make_path_chooser[browser](browser, profile_root, profiles)
except FileNotFoundError as excep:
invalid_path = exceptions_handling.invalid_path_in_tree(excep.filename)
# print(f'In {excep.filename},\npath {invalid_path} does not exist.\nMoving on...')
raise excep
else:
profilepaths = {browser_profile: str(profile_path)
for browser_profile, profile_path
in profilepaths.items()
}
return profilepaths
50 changes: 41 additions & 9 deletions united_states_of_browsers/db_merge/db_merge.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
To create a merged database, run:
$python db_merge.py
"""
import json
import os
import sqlite3
import warnings
Expand All @@ -18,19 +19,20 @@

def make_paths(app_path: [PathInfo, List[Text]],
db_name: Text,
) -> Tuple[PathInfo, PathInfo]:
) -> Tuple[PathInfo, PathInfo, PathInfo]:
""" Creates paths for app & final databse file
:param app_path: Path to directory where app data will be stored
:param db_name: Name of the database file
:return: app_path, output_db
:return: app_path, output_db_path, config_file_path
"""
try:
app_path = Path(app_path).expanduser()
except TypeError:
app_path = Path(*app_path).expanduser()
app_path.mkdir(parents=True, exist_ok=True)
output_db = app_path.joinpath(db_name)
return app_path, output_db
output_db_path = app_path.joinpath(db_name)
config_file_dirpath = Path(app_path, 'AppData')
config_file_path = Path(config_file_dirpath, 'usb_config.json')
return app_path, output_db_path, config_file_path


def find_installed_browsers(browser_info: BrowserData) -> BrowserData:
Expand Down Expand Up @@ -156,6 +158,26 @@ def write_db_path_to_file(output_db: PathInfo,
return db_path_store


def browser_info_to_json(browsers_info, config_file):
browsers_info_dict = [browser_info_._asdict() for browser_info_ in
browsers_info]
with open(config_file, 'w') as f:
json.dump(browsers_info_dict, f, default=str, indent='\t', )


def json_to_browser_info(config_file):
with open (config_file) as f:
browser_info = json.load(f)

for browser_datum in browser_info:
browser_datum['path'] = Path(browser_datum['path'])

browser_info = [BrowserData(**browser_datum)
for browser_datum in browser_info
]
return browser_info


def orchestrate_db_merge(app_path: PathInfo,
db_name: Text,
browser_info: BrowserData) -> PathInfo:
Expand All @@ -166,10 +188,20 @@ def orchestrate_db_merge(app_path: PathInfo,
:param browsers_data: NamedTuple of discovered browsers, from find_installed_browsers().
:return: Full path to final databse file.
"""
app_path, output_db = make_paths(app_path, db_name)
installed_browsers_data = find_installed_browsers(
browser_info=browser_info
)
app_path, output_db, config_filepath = make_paths(app_path, db_name)
config_filepath.parent.mkdir(parents=True, exist_ok=True)
try:
installed_browsers_data = json_to_browser_info(config_filepath)
except FileNotFoundError:
print('Searching for browsers...')
installed_browsers_data = find_installed_browsers(
browser_info=browser_info
)
browser_info_to_json(browsers_info=installed_browsers_data,
config_file=config_filepath,
)
else:
print('Reading existing configuration....')
browser_records_yielder = make_records_yielders(
browsers_data=installed_browsers_data,
app_path=app_path,
Expand Down